[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1363 - Still unstable

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
16 messages Options
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1363 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1363/

6 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node4":{             "core":"collMinRf_1x3_shard1_replica_n1",             "base_url":"http://127.0.0.1:43595/_/t",             "node_name":"127.0.0.1:43595__%2Ft",             "state":"active",             "type":"NRT"},           "core_node5":{             "core":"collMinRf_1x3_shard1_replica_n2",             "base_url":"http://127.0.0.1:34265/_/t",             "node_name":"127.0.0.1:34265__%2Ft",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node6":{             "core":"collMinRf_1x3_shard1_replica_n3",             "base_url":"http://127.0.0.1:43409/_/t",             "node_name":"127.0.0.1:43409__%2Ft",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"3",     "tlogReplicas":"0"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node4":{             "core":"collection1_shard1_replica_n3",             "base_url":"http://127.0.0.1:43595/_/t",             "node_name":"127.0.0.1:43595__%2Ft",             "state":"active",             "type":"NRT",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node2":{             "core":"collection1_shard2_replica_n1",             "base_url":"http://127.0.0.1:43409/_/t",             "node_name":"127.0.0.1:43409__%2Ft",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node6":{             "core":"collection1_shard2_replica_n5",             "base_url":"http://127.0.0.1:34265/_/t",             "node_name":"127.0.0.1:34265__%2Ft",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:35120/_/t",             "node_name":"127.0.0.1:35120__%2Ft",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node3":{             "state":"down",             "base_url":"http://127.0.0.1:35120/_/t",             "core":"c8n_1x2_shard1_replica_n2",             "node_name":"127.0.0.1:35120__%2Ft",             "type":"NRT"},           "core_node4":{             "core":"c8n_1x2_shard1_replica_n1",             "base_url":"http://127.0.0.1:43409/_/t",             "node_name":"127.0.0.1:43409__%2Ft",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"2",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collMinRf_1x3_shard1_replica_n1",
            "base_url":"http://127.0.0.1:43595/_/t",
            "node_name":"127.0.0.1:43595__%2Ft",
            "state":"active",
            "type":"NRT"},
          "core_node5":{
            "core":"collMinRf_1x3_shard1_replica_n2",
            "base_url":"http://127.0.0.1:34265/_/t",
            "node_name":"127.0.0.1:34265__%2Ft",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node6":{
            "core":"collMinRf_1x3_shard1_replica_n3",
            "base_url":"http://127.0.0.1:43409/_/t",
            "node_name":"127.0.0.1:43409__%2Ft",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n3",
            "base_url":"http://127.0.0.1:43595/_/t",
            "node_name":"127.0.0.1:43595__%2Ft",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard2_replica_n1",
            "base_url":"http://127.0.0.1:43409/_/t",
            "node_name":"127.0.0.1:43409__%2Ft",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "base_url":"http://127.0.0.1:34265/_/t",
            "node_name":"127.0.0.1:34265__%2Ft",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:35120/_/t",
            "node_name":"127.0.0.1:35120__%2Ft",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "state":"down",
            "base_url":"http://127.0.0.1:35120/_/t",
            "core":"c8n_1x2_shard1_replica_n2",
            "node_name":"127.0.0.1:35120__%2Ft",
            "type":"NRT"},
          "core_node4":{
            "core":"c8n_1x2_shard1_replica_n1",
            "base_url":"http://127.0.0.1:43409/_/t",
            "node_name":"127.0.0.1:43409__%2Ft",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"2",
    "tlogReplicas":"0"}}
        at __randomizedtesting.SeedInfo.seed([B1CF83AC33C737E6:399BBC769D3B5A1E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1992)
        at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:367)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:131)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
shard1 is not consistent.  Got 367 from https://127.0.0.1:45441/collection1_shard1_replica_n1 (previous client) and got 22 from https://127.0.0.1:40482/collection1_shard1_replica_n3

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 367 from https://127.0.0.1:45441/collection1_shard1_replica_n1 (previous client) and got 22 from https://127.0.0.1:40482/collection1_shard1_replica_n3
        at __randomizedtesting.SeedInfo.seed([B1CF83AC33C737E6:399BBC769D3B5A1E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1322)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1301)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:155)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.RestartWhileUpdatingTest:     1) Thread[id=10166, name=searcherExecutor-3247-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=10137, name=searcherExecutor-3236-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.RestartWhileUpdatingTest:
   1) Thread[id=10166, name=searcherExecutor-3247-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=10137, name=searcherExecutor-3236-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([B1CF83AC33C737E6]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=10166, name=searcherExecutor-3247-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=10137, name=searcherExecutor-3236-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=10166, name=searcherExecutor-3247-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=10137, name=searcherExecutor-3236-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([B1CF83AC33C737E6]:0)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelExecutorStream

Error Message:
Error from server at https://127.0.0.1:38713/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:38713/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([8F22DA85C3DBE894:3235AF9CFAF7D5C9]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelExecutorStream(StreamExpressionTest.java:6843)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38713/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:588)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        ... 1 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([B1CF83AC33C737E6:399BBC769D3B5A1E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11760 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> 698438 INFO  (SUITE-HttpPartitionTest-seed#[B1CF83AC33C737E6]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/init-core-data-001
   [junit4]   2> 698438 INFO  (SUITE-HttpPartitionTest-seed#[B1CF83AC33C737E6]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 698468 INFO  (SUITE-HttpPartitionTest-seed#[B1CF83AC33C737E6]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 698468 INFO  (SUITE-HttpPartitionTest-seed#[B1CF83AC33C737E6]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_/t
   [junit4]   2> 698470 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 698484 INFO  (Thread-1064) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 698484 INFO  (Thread-1064) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 698532 ERROR (Thread-1064) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 698589 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ZkTestServer start zk server on port:45408
   [junit4]   2> 698697 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 698698 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 698712 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 698713 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 698714 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 698715 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 698744 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 698745 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 698745 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 698746 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 698747 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 698760 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 699295 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 699308 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3fc9ad1f{/_/t,null,AVAILABLE}
   [junit4]   2> 699349 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@969acd9{HTTP/1.1,[http/1.1]}{127.0.0.1:43062}
   [junit4]   2> 699349 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server Started @708214ms
   [junit4]   2> 699349 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/tempDir-001/control/data, replicaType=NRT, hostContext=/_/t, hostPort=35120, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/control-001/cores}
   [junit4]   2> 699365 ERROR (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 699430 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 699430 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 699430 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 699430 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-26T11:40:25.368Z
   [junit4]   2> 699510 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 699510 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/control-001/solr.xml
   [junit4]   2> 699567 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 699588 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45408/solr
   [junit4]   2> 699961 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 699961 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35120__%2Ft
   [junit4]   2> 699970 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.Overseer Overseer (id=98374072693030916-127.0.0.1:35120__%2Ft-n_0000000000) starting
   [junit4]   2> 699985 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35120__%2Ft
   [junit4]   2> 700004 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 700297 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 700313 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 700313 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 700314 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/control-001/cores
   [junit4]   2> 700379 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 700380 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45408/solr ready
   [junit4]   2> 700380 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46272,localport=35120], receiveBufferSize:531000
   [junit4]   2> 700381 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43062,localport=33950], receiveBufferSize=530904
   [junit4]   2> 700406 INFO  (qtp1823974227-2140) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35120__%252Ft&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 700440 INFO  (OverseerThreadFactory-530-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 700596 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46276,localport=35120], receiveBufferSize:531000
   [junit4]   2> 700612 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43062,localport=33954], receiveBufferSize=530904
   [junit4]   2> 700678 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 700678 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 700905 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 701914 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 701993 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 702459 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 702501 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 702514 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 702514 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 702514 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 702517 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=31.7314453125, floorSegmentMB=1.115234375, forceMergeDeletesPctAllowed=7.178937371096712, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 702528 WARN  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 702689 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 702689 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 702690 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 702690 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 702691 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1207789220, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 702692 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@597f69c[control_collection_shard1_replica_n1] main]
   [junit4]   2> 702692 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 702693 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 702693 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 702771 INFO  (searcherExecutor-533-thread-1-processing-n:127.0.0.1:35120__%2Ft x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@597f69c[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 702771 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573985167562768384
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35120/_/t/control_collection_shard1_replica_n1/
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35120/_/t/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 702820 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 702853 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35120/_/t/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 702959 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 703003 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 703007 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2329
   [junit4]   2> 703039 INFO  (qtp1823974227-2140) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 703132 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 704046 INFO  (qtp1823974227-2140) [n:127.0.0.1:35120__%2Ft    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35120__%252Ft&wt=javabin&version=2} status=0 QTime=3640
   [junit4]   2> 704061 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 704061 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45408/solr ready
   [junit4]   2> 704061 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 704070 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46282,localport=35120], receiveBufferSize:531000
   [junit4]   2> 704070 INFO  (SocketProxy-Acceptor-35120) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43062,localport=33960], receiveBufferSize=530904
   [junit4]   2> 704083 INFO  (qtp1823974227-2142) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 704085 INFO  (OverseerThreadFactory-530-thread-2-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 704086 INFO  (OverseerCollectionConfigSetProcessor-98374072693030916-127.0.0.1:35120__%2Ft-n_0000000000) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 704087 WARN  (OverseerThreadFactory-530-thread-2-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 704331 INFO  (qtp1823974227-2142) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 704331 INFO  (qtp1823974227-2142) [n:127.0.0.1:35120__%2Ft    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=248
   [junit4]   2> 704640 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001 of type NRT
   [junit4]   2> 704641 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 704641 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3131c5c3{/_/t,null,AVAILABLE}
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@28a339f8{HTTP/1.1,[http/1.1]}{127.0.0.1:36721}
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server Started @713507ms
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/_/t, hostPort=43409, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001/cores}
   [junit4]   2> 704642 ERROR (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 704642 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-26T11:40:30.580Z
   [junit4]   2> 704658 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 704658 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001/solr.xml
   [junit4]   2> 704675 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 704724 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45408/solr
   [junit4]   2> 704770 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 704771 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 704772 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43409__%2Ft
   [junit4]   2> 704773 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 704774 INFO  (zkCallback-223-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 704776 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 705008 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 705024 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 705024 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 705038 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001/cores
   [junit4]   2> 705114 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43409__%252Ft&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 705144 INFO  (OverseerCollectionConfigSetProcessor-98374072693030916-127.0.0.1:35120__%2Ft-n_0000000000) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 705144 INFO  (OverseerThreadFactory-530-thread-3-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:43409__%2Ft for creating new replica
   [junit4]   2> 705147 INFO  (SocketProxy-Acceptor-43409) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42942,localport=43409], receiveBufferSize:531000
   [junit4]   2> 705253 INFO  (SocketProxy-Acceptor-43409) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36721,localport=40156], receiveBufferSize=530904
   [junit4]   2> 705253 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 705254 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 705399 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 706331 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 706395 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
   [junit4]   2> 706921 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 706986 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 706986 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 706986 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 706986 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001/cores/collection1_shard2_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-1-001/cores/collection1_shard2_replica_n1/data/]
   [junit4]   2> 707002 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=31.7314453125, floorSegmentMB=1.115234375, forceMergeDeletesPctAllowed=7.178937371096712, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 707018 WARN  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 707280 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 707280 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 707281 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 707281 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 707431 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1207789220, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 707431 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@81c52ff[collection1_shard2_replica_n1] main]
   [junit4]   2> 707432 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 707432 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 707433 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 707433 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573985172451229696
   [junit4]   2> 707520 INFO  (searcherExecutor-544-thread-1-processing-n:127.0.0.1:43409__%2Ft x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@81c52ff[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 707550 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 707551 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 707551 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43409/_/t/collection1_shard2_replica_n1/
   [junit4]   2> 707551 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 707551 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43409/_/t/collection1_shard2_replica_n1/ has no replicas
   [junit4]   2> 707551 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 707565 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43409/_/t/collection1_shard2_replica_n1/ shard2
   [junit4]   2> 707566 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 707703 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 707723 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 707725 INFO  (qtp1491927739-2189) [n:127.0.0.1:43409__%2Ft c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2471
   [junit4]   2> 707757 INFO  (qtp1823974227-2138) [n:127.0.0.1:35120__%2Ft    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43409__%252Ft&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=2642
   [junit4]   2> 707837 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 708180 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001 of type NRT
   [junit4]   2> 708181 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 708280 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@8ebf188{/_/t,null,AVAILABLE}
   [junit4]   2> 708280 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@cb2a64d{HTTP/1.1,[http/1.1]}{127.0.0.1:41239}
   [junit4]   2> 708280 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.e.j.s.Server Started @717146ms
   [junit4]   2> 708280 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/_/t, hostPort=43595, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001/cores}
   [junit4]   2> 708281 ERROR (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 708284 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 708284 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 708284 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 708284 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-26T11:40:34.222Z
   [junit4]   2> 708326 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 708326 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001/solr.xml
   [junit4]   2> 708371 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 708418 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45408/solr
   [junit4]   2> 708563 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 708576 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 708579 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43595__%2Ft
   [junit4]   2> 708580 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 708658 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:43595__%2Ft) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 708580 INFO  (zkCallback-216-thread-1-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 708675 INFO  (zkCallback-223-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 709149 INFO  (OverseerCollectionConfigSetProcessor-98374072693030916-127.0.0.1:35120__%2Ft-n_0000000000) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 709165 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 709211 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 709211 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 709212 INFO  (TEST-HttpPartitionTest.test-seed#[B1CF83AC33C737E6]) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001/cores
   [junit4]   2> 709429 INFO  (qtp1823974227-2140) [n:127.0.0.1:35120__%2Ft    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43595__%252Ft&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 709446 INFO  (OverseerThreadFactory-530-thread-4-processing-n:127.0.0.1:35120__%2Ft) [n:127.0.0.1:35120__%2Ft    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:43595__%2Ft for creating new replica
   [junit4]   2> 709448 INFO  (SocketProxy-Acceptor-43595) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48538,localport=43595], receiveBufferSize:531000
   [junit4]   2> 709495 INFO  (SocketProxy-Acceptor-43595) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41239,localport=33532], receiveBufferSize=530904
   [junit4]   2> 709497 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 709498 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 709645 INFO  (zkCallback-228-thread-1-processing-n:127.0.0.1:43409__%2Ft) [n:127.0.0.1:43409__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 709645 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:43595__%2Ft) [n:127.0.0.1:43595__%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 710598 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 710678 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema [collection1_shard1_replica_n3] Schema name=test
   [junit4]   2> 711155 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 711248 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n3' using configuration from collection collection1, trusted=true
   [junit4]   2> 711248 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n3' (registry 'solr.core.collection1.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b53d75
   [junit4]   2> 711249 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 711249 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [[collection1_shard1_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001/cores/collection1_shard1_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_B1CF83AC33C737E6-001/shard-2-001/cores/collection1_shard1_replica_n3/data/]
   [junit4]   2> 711263 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=31.7314453125, floorSegmentMB=1.115234375, forceMergeDeletesPctAllowed=7.178937371096712, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 711265 WARN  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 711487 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 711487 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 711488 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 711488 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 711489 INFO  (qtp2054927843-2221) [n:127.0.0.1:43595__%2Ft c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy Rand

[...truncated too long message...]

1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.workQueue.shard1.leader, tag=566370447
   [junit4]   2> 174427 INFO  (coreCloseExecutor-404-thread-2) [n:127.0.0.1:44682_solr c:workQueue s:shard2 r:core_node4 x:workQueue_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.workQueue.shard2.leader, tag=859387895
   [junit4]   2> 174427 INFO  (coreCloseExecutor-404-thread-3) [n:127.0.0.1:44682_solr c:mainCorpus s:shard1 r:core_node3 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.mainCorpus.shard1.replica_n1, tag=1033337030
   [junit4]   2> 174428 WARN  (zkCallback-727-thread-2-processing-n:127.0.0.1:45204_solr) [n:127.0.0.1:45204_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 174428 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:39636_solr) [n:127.0.0.1:39636_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 174429 INFO  (zkCallback-738-thread-2-processing-n:127.0.0.1:44682_solr) [n:127.0.0.1:44682_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 174429 INFO  (jetty-closer-715-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@799d969b{/solr,null,UNAVAILABLE}
   [junit4]   2> 174429 INFO  (coreCloseExecutor-406-thread-1) [n:127.0.0.1:39636_solr c:mainCorpus s:shard2 r:core_node4 x:mainCorpus_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard2.leader, tag=1663320135
   [junit4]   2> 174430 INFO  (coreCloseExecutor-406-thread-2) [n:127.0.0.1:39636_solr c:destination s:shard2 r:core_node4 x:destination_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.destination.shard2.leader, tag=2028784882
   [junit4]   2> 174430 INFO  (coreCloseExecutor-404-thread-3) [n:127.0.0.1:44682_solr c:mainCorpus s:shard1 r:core_node3 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard1.leader, tag=1033337030
   [junit4]   2> 174430 INFO  (coreCloseExecutor-404-thread-4) [n:127.0.0.1:44682_solr c:destination s:shard1 r:core_node3 x:destination_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.destination.shard1.replica_n1, tag=487337752
   [junit4]   2> 174431 INFO  (jetty-closer-715-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98374316573982728-127.0.0.1:39636_solr-n_0000000000) closing
   [junit4]   2> 174431 INFO  (OverseerStateUpdate-98374316573982728-127.0.0.1:39636_solr-n_0000000000) [n:127.0.0.1:39636_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39636_solr
   [junit4]   2> 174432 INFO  (coreCloseExecutor-404-thread-4) [n:127.0.0.1:44682_solr c:destination s:shard1 r:core_node3 x:destination_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.destination.shard1.leader, tag=487337752
   [junit4]   2> 174434 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d7eeb726a000c, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 175903 WARN  (zkCallback-732-thread-1-processing-n:127.0.0.1:38713_solr) [n:127.0.0.1:38713_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175903 INFO  (jetty-closer-715-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2cc94211{/solr,null,UNAVAILABLE}
   [junit4]   2> 175933 WARN  (zkCallback-731-thread-1-processing-n:127.0.0.1:39636_solr) [n:127.0.0.1:39636_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175934 WARN  (zkCallback-738-thread-1-processing-n:127.0.0.1:44682_solr) [n:127.0.0.1:44682_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175934 INFO  (jetty-closer-715-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39c2fe9d{/solr,null,UNAVAILABLE}
   [junit4]   2> 175934 INFO  (jetty-closer-715-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7cdf0e39{/solr,null,UNAVAILABLE}
   [junit4]   2> 175934 ERROR (SUITE-StreamExpressionTest-seed#[8F22DA85C3DBE894]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 175934 INFO  (SUITE-StreamExpressionTest-seed#[8F22DA85C3DBE894]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45131 45131
   [junit4]   2> 176047 INFO  (Thread-178) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45131 45131
   [junit4]   2> 176048 WARN  (Thread-178) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 68 /solr/aliases.json
   [junit4]   2> 6 /solr/configs/conf
   [junit4]   2> 4 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 68 /solr/clusterprops.json
   [junit4]   2> 68 /solr/clusterstate.json
   [junit4]   2> 20 /solr/collections/destination/state.json
   [junit4]   2> 20 /solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 20 /solr/collections/workQueue/state.json
   [junit4]   2> 20 /solr/collections/destinationCollection/state.json
   [junit4]   2> 20 /solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 20 /solr/collections/mainCorpus/state.json
   [junit4]   2> 10 /solr/collections/collection1_collection/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98374316573982728-127.0.0.1:39636_solr-n_0000000000
   [junit4]   2> 2 /solr/overseer_elect/election/98374316573982726-127.0.0.1:45204_solr-n_0000000001
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 68 /solr/live_nodes
   [junit4]   2> 68 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.io.stream.StreamExpressionTest_8F22DA85C3DBE894-001
   [junit4]   2> Jul 26, 2017 12:43:45 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {expr_s=FSTOrd50, join2_s=PostingsFormat(name=Memory), multiDefault=PostingsFormat(name=Memory), subject=PostingsFormat(name=Memory), a1_s=PostingsFormat(name=Memory), a2_s=FST50, level1_s=FSTOrd50, test_t=FST50, body_t=FST50, ident_s=PostingsFormat(name=Memory), s_multi=FST50, level2_s=PostingsFormat(name=LuceneFixedGap), a_ss=PostingsFormat(name=LuceneFixedGap), col_s=PostingsFormat(name=Memory), a_s=FST50, a_t=PostingsFormat(name=LuceneFixedGap), side_s=FST50, id=FSTOrd50, text=FST50}, docValues:{expr_s=DocValuesFormat(name=Lucene70), join2_s=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Memory), price_f=DocValuesFormat(name=Memory), a1_s=DocValuesFormat(name=Memory), a_f=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Direct), a2_s=DocValuesFormat(name=Asserting), join1_i=DocValuesFormat(name=Lucene70), a_i=DocValuesFormat(name=Lucene70), test_i=DocValuesFormat(name=Memory), join3_i=DocValuesFormat(name=Asserting), field_i=DocValuesFormat(name=Memory), level1_s=DocValuesFormat(name=Lucene70), s_multi=DocValuesFormat(name=Asserting), level2_s=DocValuesFormat(name=Direct), a_ss=DocValuesFormat(name=Direct), col_s=DocValuesFormat(name=Memory), a_s=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Direct), i_multi=DocValuesFormat(name=Lucene70), b_ls=DocValuesFormat(name=Memory), test_dt=DocValuesFormat(name=Lucene70), ident_s=DocValuesFormat(name=Memory), order_i=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Direct), side_s=DocValuesFormat(name=Asserting), miles_i=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=280, maxMBSortInHeap=6.403316229369698, sim=RandomSimilarity(queryNorm=false): {}, locale=ca, timezone=Africa/Casablanca
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=77546424,total=398458880
   [junit4]   2> NOTE: All tests run in this JVM: [CoalesceEvaluatorTest, SolrExampleJettyTest, LargeVolumeJettyTest, DocumentAnalysisResponseTest, StreamingTest, ModifiableSolrParamsTest, NamedListTest, ConcurrentUpdateSolrClientBuilderTest, SolrZkClientTest, SolrPingTest, AndEvaluatorTest, SquareRootEvaluatorTest, SolrParamTest, HttpSolrClientConPoolTest, JdbcTest, SolrExceptionTest, TestV2Request, NotEvaluatorTest, SolrSchemalessExampleTest, GraphExpressionTest, TestDelegationTokenRequest, TestBatchUpdate, CloudSolrClientBuilderTest, CloudSolrClientCacheTest, LBHttpSolrClientBuilderTest, JdbcDriverTest, StreamExpressionTest]
   [junit4] Completed [135/135 (1!)] on J2 in 80.46s, 92 tests, 1 error, 9 skipped <<< FAILURES!

[...truncated 42388 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1364 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1364/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:     1) Thread[id=18393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=18411, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=18402, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:
   1) Thread[id=18393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=18411, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=18402, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([2364B22A7758F37A]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node3":{             "core":"collMinRf_1x3_shard1_replica_n1",             "base_url":"http://127.0.0.1:40405/e_/at",             "node_name":"127.0.0.1:40405_e_%2Fat",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node5":{             "core":"collMinRf_1x3_shard1_replica_n2",             "base_url":"http://127.0.0.1:55221/e_/at",             "node_name":"127.0.0.1:55221_e_%2Fat",             "state":"active",             "type":"NRT"},           "core_node6":{             "core":"collMinRf_1x3_shard1_replica_n4",             "base_url":"http://127.0.0.1:38496/e_/at",             "node_name":"127.0.0.1:38496_e_%2Fat",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"3",     "tlogReplicas":"0"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node4":{             "core":"collection1_shard1_replica_n3",             "base_url":"http://127.0.0.1:55221/e_/at",             "node_name":"127.0.0.1:55221_e_%2Fat",             "state":"active",             "type":"NRT",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node2":{             "core":"collection1_shard2_replica_n1",             "base_url":"http://127.0.0.1:38496/e_/at",             "node_name":"127.0.0.1:38496_e_%2Fat",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node6":{             "core":"collection1_shard2_replica_n5",             "base_url":"http://127.0.0.1:39222/e_/at",             "node_name":"127.0.0.1:39222_e_%2Fat",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:40405/e_/at",             "node_name":"127.0.0.1:40405_e_%2Fat",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node3":{             "core":"c8n_1x2_shard1_replica_n1",             "base_url":"http://127.0.0.1:55221/e_/at",             "node_name":"127.0.0.1:55221_e_%2Fat",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node4":{             "state":"down",             "base_url":"http://127.0.0.1:38496/e_/at",             "core":"c8n_1x2_shard1_replica_n2",             "node_name":"127.0.0.1:38496_e_%2Fat",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"2",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collMinRf_1x3_shard1_replica_n1",
            "base_url":"http://127.0.0.1:40405/e_/at",
            "node_name":"127.0.0.1:40405_e_%2Fat",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node5":{
            "core":"collMinRf_1x3_shard1_replica_n2",
            "base_url":"http://127.0.0.1:55221/e_/at",
            "node_name":"127.0.0.1:55221_e_%2Fat",
            "state":"active",
            "type":"NRT"},
          "core_node6":{
            "core":"collMinRf_1x3_shard1_replica_n4",
            "base_url":"http://127.0.0.1:38496/e_/at",
            "node_name":"127.0.0.1:38496_e_%2Fat",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n3",
            "base_url":"http://127.0.0.1:55221/e_/at",
            "node_name":"127.0.0.1:55221_e_%2Fat",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard2_replica_n1",
            "base_url":"http://127.0.0.1:38496/e_/at",
            "node_name":"127.0.0.1:38496_e_%2Fat",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "base_url":"http://127.0.0.1:39222/e_/at",
            "node_name":"127.0.0.1:39222_e_%2Fat",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:40405/e_/at",
            "node_name":"127.0.0.1:40405_e_%2Fat",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x2_shard1_replica_n1",
            "base_url":"http://127.0.0.1:55221/e_/at",
            "node_name":"127.0.0.1:55221_e_%2Fat",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node4":{
            "state":"down",
            "base_url":"http://127.0.0.1:38496/e_/at",
            "core":"c8n_1x2_shard1_replica_n2",
            "node_name":"127.0.0.1:38496_e_%2Fat",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"2",
    "tlogReplicas":"0"}}
        at __randomizedtesting.SeedInfo.seed([2364B22A7758F37A:AB308DF0D9A49E82]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1992)
        at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:367)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:131)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([2364B22A7758F37A:3F65CFA702FD8DE9]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertFalse(Assert.java:68)
        at org.junit.Assert.assertFalse(Assert.java:79)
        at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:497)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:59629/q_rck: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59629/q_rck: ADDREPLICA failed to create replica
        at __randomizedtesting.SeedInfo.seed([2364B22A7758F37A:AB308DF0D9A49E82]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:801)
        at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:374)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([2364B22A7758F37A:AB308DF0D9A49E82]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12066 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTlogReplica
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/init-core-data-001
   [junit4]   2> 948710 WARN  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 948712 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 948714 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 948714 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 948715 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001
   [junit4]   2> 948715 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 948715 INFO  (Thread-5628) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 948715 INFO  (Thread-5628) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 948726 ERROR (Thread-5628) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 948815 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:45243
   [junit4]   2> 948834 INFO  (jetty-launcher-1215-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 948835 INFO  (jetty-launcher-1215-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 948836 INFO  (jetty-launcher-1215-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ad78eda{/solr,null,AVAILABLE}
   [junit4]   2> 948837 INFO  (jetty-launcher-1215-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16bce4fb{/solr,null,AVAILABLE}
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@782fa865{HTTP/1.1,[http/1.1]}{127.0.0.1:41405}
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-1) [    ] o.e.j.s.Server Started @951878ms
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41405}
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2e06b712{HTTP/1.1,[http/1.1]}{127.0.0.1:54905}
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-2) [    ] o.e.j.s.Server Started @951879ms
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54905}
   [junit4]   2> 948844 ERROR (jetty-launcher-1215-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 948844 ERROR (jetty-launcher-1215-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 948844 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T10:32:27.893Z
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 948845 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-27T10:32:27.893Z
   [junit4]   2> 948848 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 948858 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 948866 INFO  (jetty-launcher-1215-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45243/solr
   [junit4]   2> 948867 INFO  (jetty-launcher-1215-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45243/solr
   [junit4]   2> 948872 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d839ad1770004, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 949000 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 949000 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41405_solr
   [junit4]   2> 949001 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.c.Overseer Overseer (id=98379467834458118-127.0.0.1:41405_solr-n_0000000000) starting
   [junit4]   2> 949032 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41405_solr
   [junit4]   2> 949051 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 949053 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 949055 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 949069 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54905_solr
   [junit4]   2> 949072 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 949075 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 949141 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41405.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949168 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41405.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949168 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41405.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949169 INFO  (jetty-launcher-1215-thread-1) [n:127.0.0.1:41405_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node1/.
   [junit4]   2> 949174 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54905.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949179 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54905.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949179 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54905.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 949180 INFO  (jetty-launcher-1215-thread-2) [n:127.0.0.1:54905_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node2/.
   [junit4]   2> 949234 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 949236 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45243/solr ready
   [junit4]   2> 949240 INFO  (SUITE-TestTlogReplica-seed#[2364B22A7758F37A]-worker) [    ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
   [junit4]   2> 949250 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 949251 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 949429 INFO  (TEST-TestTlogReplica.testDeleteById-seed#[2364B22A7758F37A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteById
   [junit4]   2> 949432 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_delete_by_id&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 949434 INFO  (OverseerThreadFactory-2757-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_delete_by_id
   [junit4]   2> 949543 INFO  (OverseerStateUpdate-98379467834458118-127.0.0.1:41405_solr-n_0000000000) [n:127.0.0.1:41405_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_delete_by_id",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_delete_by_id_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:54905/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 949546 INFO  (OverseerStateUpdate-98379467834458118-127.0.0.1:41405_solr-n_0000000000) [n:127.0.0.1:41405_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_delete_by_id",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_delete_by_id_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:41405/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 949762 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_delete_by_id_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_delete_by_id&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 949763 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 949771 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_delete_by_id_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_delete_by_id&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 949772 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 949881 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 949881 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 949882 INFO  (zkCallback-1227-thread-2-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 949882 INFO  (zkCallback-1226-thread-2-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 950799 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 950804 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 950808 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_delete_by_id_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 950811 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 950811 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_delete_by_id_shard1_replica_t2' using configuration from collection tlog_replica_test_delete_by_id, trusted=true
   [junit4]   2> 950812 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41405.solr.core.tlog_replica_test_delete_by_id.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_delete_by_id.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 950812 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 950812 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_delete_by_id_shard1_replica_t2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node1/tlog_replica_test_delete_by_id_shard1_replica_t2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node1/./tlog_replica_test_delete_by_id_shard1_replica_t2/data/]
   [junit4]   2> 950813 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_delete_by_id_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 950815 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 950816 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_delete_by_id_shard1_replica_t1' using configuration from collection tlog_replica_test_delete_by_id, trusted=true
   [junit4]   2> 950816 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54905.solr.core.tlog_replica_test_delete_by_id.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_delete_by_id.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@356d68aa
   [junit4]   2> 950817 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 950817 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_delete_by_id_shard1_replica_t1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node2/tlog_replica_test_delete_by_id_shard1_replica_t1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2364B22A7758F37A-001/tempDir-001/node2/./tlog_replica_test_delete_by_id_shard1_replica_t1/data/]
   [junit4]   2> 950890 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 950890 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 950892 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 950892 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 950897 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@107ac929[tlog_replica_test_delete_by_id_shard1_replica_t2] main]
   [junit4]   2> 950899 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 950900 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 950900 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 950901 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574071487639322624
   [junit4]   2> 950908 INFO  (searcherExecutor-2762-thread-1-processing-n:127.0.0.1:41405_solr x:tlog_replica_test_delete_by_id_shard1_replica_t2 s:shard1 c:tlog_replica_test_delete_by_id r:core_node4) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_delete_by_id_shard1_replica_t2] Registered new searcher Searcher@107ac929[tlog_replica_test_delete_by_id_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 950917 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 950917 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 950917 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 950919 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 950919 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 950921 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@34eeed83[tlog_replica_test_delete_by_id_shard1_replica_t1] main]
   [junit4]   2> 950922 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 950923 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 950924 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 950927 INFO  (searcherExecutor-2763-thread-1-processing-n:127.0.0.1:54905_solr x:tlog_replica_test_delete_by_id_shard1_replica_t1 s:shard1 c:tlog_replica_test_delete_by_id r:core_node3) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_delete_by_id_shard1_replica_t1] Registered new searcher Searcher@34eeed83[tlog_replica_test_delete_by_id_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 950927 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574071487666585600
   [junit4]   2> 951018 INFO  (zkCallback-1227-thread-2-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951018 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951018 INFO  (zkCallback-1226-thread-2-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951018 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951418 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 951418 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 951418 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/
   [junit4]   2> 951419 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_delete_by_id_shard1_replica_t2 url=http://127.0.0.1:41405/solr START replicas=[http://127.0.0.1:54905/solr/tlog_replica_test_delete_by_id_shard1_replica_t1/] nUpdates=100
   [junit4]   2> 951419 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_delete_by_id_shard1_replica_t2 url=http://127.0.0.1:41405/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 951428 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 951429 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 951429 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 951429 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 951429 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_delete_by_id_shard1_replica_t2 stopping background replication from leader
   [junit4]   2> 951433 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/ shard1
   [junit4]   2> 951537 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951537 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951537 INFO  (zkCallback-1227-thread-2-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951537 INFO  (zkCallback-1226-thread-2-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951584 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 951586 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_delete_by_id_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_delete_by_id&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1823
   [junit4]   2> 951689 INFO  (zkCallback-1226-thread-2-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951689 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951690 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951690 INFO  (zkCallback-1227-thread-2-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 951938 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_delete_by_id_shard1_replica_t1 starting background replication from leader
   [junit4]   2> 951938 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 951941 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 951943 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_delete_by_id_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_delete_by_id&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2171
   [junit4]   2> 951951 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 952051 INFO  (zkCallback-1226-thread-2-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 952052 INFO  (zkCallback-1227-thread-2-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 952052 INFO  (zkCallback-1226-thread-1-processing-n:127.0.0.1:54905_solr) [n:127.0.0.1:54905_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 952052 INFO  (zkCallback-1227-thread-1-processing-n:127.0.0.1:41405_solr) [n:127.0.0.1:41405_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_delete_by_id/state.json] for collection [tlog_replica_test_delete_by_id] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 952580 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/
   [junit4]   2> 952591 INFO  (qtp365163325-11849) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=2
   [junit4]   2> 952592 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 952592 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 952592 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 952592 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 952951 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_delete_by_id&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=3519
   [junit4]   2> 952993 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1574071489799389184&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&wt=javabin&version=2}{deleteByQuery=*:* (-1574071489799389184)} 0 0
   [junit4]   2> 952995 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1574071489835040768,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 952995 INFO  (qtp1081688247-11847) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 952995 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 952995 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 952996 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 952996 INFO  (qtp365163325-11853) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/update params={_stateVer_=tlog_replica_test_delete_by_id:6&commit=true&wt=javabin&version=2}{deleteByQuery=*:* (-1574071489799389184),commit=} 0 35
   [junit4]   2> 953005 INFO  (qtp1081688247-11845) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&wt=javabin&version=2}{add=[1 (1574071489840283648)]} 0 0
   [junit4]   2> 953005 INFO  (qtp365163325-11843) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/update params={_stateVer_=tlog_replica_test_delete_by_id:6&wt=javabin&version=2}{add=[1 (1574071489840283648)]} 0 5
   [junit4]   2> 953006 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 953007 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1574071489847623680,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 953013 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1f3d3a commitCommandVersion:1574071489847623680
   [junit4]   2> 953025 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@47fe36b7[tlog_replica_test_delete_by_id_shard1_replica_t2] main]
   [junit4]   2> 953025 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 953026 INFO  (searcherExecutor-2762-thread-1-processing-n:127.0.0.1:41405_solr x:tlog_replica_test_delete_by_id_shard1_replica_t2 s:shard1 c:tlog_replica_test_delete_by_id r:core_node4) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_delete_by_id_shard1_replica_t2] Registered new searcher Searcher@47fe36b7[tlog_replica_test_delete_by_id_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))}
   [junit4]   2> 953027 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:41405/solr/tlog_replica_test_delete_by_id_shard1_replica_t2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 20
   [junit4]   2> 953027 INFO  (qtp365163325-11849) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/update params={_stateVer_=tlog_replica_test_delete_by_id:6&commit=true&wt=javabin&version=2}{commit=} 0 21
   [junit4]   2> 953030 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953131 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953233 INFO  (qtp1081688247-11847) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953334 INFO  (qtp1081688247-11845) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953435 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953438 INFO  (OverseerCollectionConfigSetProcessor-98379467834458118-127.0.0.1:41405_solr-n_0000000000) [n:127.0.0.1:41405_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 953536 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953637 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953747 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953848 INFO  (qtp1081688247-11847) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 953949 INFO  (qtp1081688247-11847) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954050 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954151 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954253 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954354 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954455 INFO  (qtp1081688247-11845) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954556 INFO  (qtp1081688247-11847) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954657 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954759 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954860 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 954961 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955064 INFO  (qtp1081688247-11845) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955165 INFO  (qtp1081688247-11845) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955266 INFO  (qtp1081688247-11857) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955367 INFO  (qtp1081688247-11854) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955468 INFO  (qtp1081688247-11858) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955569 INFO  (qtp1081688247-11850) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 955580 INFO  (qtp365163325-11842) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 955581 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 955581 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 1501151552061
   [junit4]   2> 955581 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 955581 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 955581 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 955582 INFO  (qtp365163325-11846) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=0
   [junit4]   2> 955582 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Number of files in latest index in master: 16
   [junit4]   2> 955595 INFO  (indexFetcher-2773-thread-1) [n:127.0.0.1:54905_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node3 x:tlog_replica_test_delete_by_id_shard1_replica_t1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@12c89bb0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b70e48f)
   [junit4]   2> 955596 INFO  (qtp365163325-11853) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 955599 INFO  (qtp365163325-11843) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 955603 INFO  (qtp365163325-11851) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 955604 INFO  (qtp365163325-11849) [n:127.0.0.1:41405_solr c:tlog_replica_test_delete_by_id s:shard1 r:core_node4 x:tlog_replica_test_delete_by_id_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_delete_by_id_shard1_replica_t2]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 955605 INFO  (qtp365163325-11842) [n:127.0.0.1:41405_solr c:

[...truncated too long message...]

reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
   [junit4]   2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> ... 54 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2364B22A7758F37A-001
   [junit4]   2> Jul 27, 2017 11:06:45 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=en-SG, timezone=IST
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=129562816,total=521666560
   [junit4]   2> NOTE: All tests run in this JVM: [TestIBSimilarityFactory, TestExactStatsCache, FieldAnalysisRequestHandlerTest, SyncSliceTest, TestSizeLimitedDistributedMap, ForceLeaderTest, CdcrReplicationHandlerTest, TestCrossCoreJoin, TestRealTimeGet, TestNumericTokenStream, OverseerRolesTest, RollingRestartTest, TestDefaultStatsCache, TestClusterStateMutator, CollectionStateFormat2Test, FullHLLTest, TestStressLiveNodes, DistributedQueryElevationComponentTest, TestQueryWrapperFilter, TestRemoteStreaming, ConnectionManagerTest, TestSolr4Spatial2, TestFaceting, DeleteInactiveReplicaTest, TestCloudPseudoReturnFields, TestFastOutputStream, AssignBackwardCompatibilityTest, TestAddFieldRealTimeGet, ClassificationUpdateProcessorIntegrationTest, TestQuerySenderNoQuery, SmileWriterTest, SparseHLLTest, SuggesterWFSTTest, TestRandomFaceting, EchoParamsTest, OverriddenZkACLAndCredentialsProvidersTest, ZkNodePropsTest, TestReplicationHandler, TestConfigSetProperties, TestCloudInspectUtil, SolrCloudReportersTest, XmlUpdateRequestHandlerTest, WrapperMergePolicyFactoryTest, CoreMergeIndexesAdminHandlerTest, TestBulkSchemaAPI, TestComplexPhraseQParserPlugin, TestConfigSetsAPIExclusivity, BlobRepositoryCloudTest, HdfsSyncSliceTest, AtomicUpdateProcessorFactoryTest, TestDelegationWithHadoopAuth, SpellCheckCollatorWithCollapseTest, PropertiesRequestHandlerTest, BigEndianAscendingWordSerializerTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, IndexBasedSpellCheckerTest, TestManagedResource, TestCollationField, RAMDirectoryFactoryTest, DistributedExpandComponentTest, TestReRankQParserPlugin, LeaderElectionIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestFunctionQuery, TestPKIAuthenticationPlugin, CollectionsAPISolrJTest, VMParamsZkACLAndCredentialsProvidersTest, DistribCursorPagingTest, TestCSVLoader, TestHighlightDedupGrouping, TestSubQueryTransformer, QueryElevationComponentTest, TestReplicationHandlerBackup, TestFieldCacheSortRandom, TestRawTransformer, TestSolrDeletionPolicy1, SolrXmlInZkTest, TestFieldCache, LeaderFailoverAfterPartitionTest, TestUnifiedSolrHighlighter, BaseCdcrDistributedZkTest, SuggestComponentContextFilterQueryTest, BinaryUpdateRequestHandlerTest, BasicDistributedZkTest, MergeStrategyTest, DateRangeFieldTest, BlockCacheTest, TestRangeQuery, TestSolrQueryParser, TestBinaryField, OutputWriterTest, SimpleMLTQParserTest, TestIndexingPerformance, TestDeleteCollectionOnDownNodes, HdfsUnloadDistributedZkTest, DistanceUnitsTest, TestMergePolicyConfig, TestSQLHandler, TestCloudNestedDocsSort, TestOverriddenPrefixQueryForCustomFieldType, V2StandaloneTest, TestConfigsApi, TestDistributedSearch, SolrIndexMetricsTest, TestDocumentBuilder, TestSearcherReuse, PeerSyncReplicationTest, TestSchemaResource, FieldMutatingUpdateProcessorTest, HdfsBasicDistributedZk2Test, TestNamedUpdateProcessors, TestCollectionAPI, UUIDFieldTest, TestDynamicFieldResource, TestSortByMinMaxFunction, TestRandomDVFaceting, SharedFSAutoReplicaFailoverTest]
   [junit4] Completed [534/726 (4!)] on J0 in 286.02s, 1 test, 1 failure <<< FAILURES!

[...truncated 43794 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1365 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1365/

3 tests failed.
FAILED:  org.apache.lucene.index.TestIndexSorting.testRandom3

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([153A273D1319441C:B7E269E777EB6D1A]:0)
        at org.apache.lucene.util.packed.Packed64.<init>(Packed64.java:73)
        at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:972)
        at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
        at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
        at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
        at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
        at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
        at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
        at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
        at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
        at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
        at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
        at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
        at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
        at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
        at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)


FAILED:  org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([DCE716222F6A0505:F0D561AFA3AE7381]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:264)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
        xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

        request was:q=int_i:17&wt=xml
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:879)
        ... 40 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([DCE716222F6A0505:54B329F8819668FD]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1230 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexSorting
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=153A273D1319441C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-MT -Dtests.timezone=Pacific/Pohnpei -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    146s J1 | TestIndexSorting.testRandom3 <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > at __randomizedtesting.SeedInfo.seed([153A273D1319441C:B7E269E777EB6D1A]:0)
   [junit4]    > at org.apache.lucene.util.packed.Packed64.<init>(Packed64.java:73)
   [junit4]    > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:972)
   [junit4]    > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
   [junit4]    > at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
   [junit4]    > at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
   [junit4]    > at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
   [junit4]    > at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
   [junit4]    > at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
   [junit4]    > at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
   [junit4]    > at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
   [junit4]    > at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
   [junit4]    > at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
   [junit4]    > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
   [junit4]    > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
   [junit4]    > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
   [junit4]    > at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
   [junit4]    > at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
   [junit4]    > at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]    > at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]    > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]    > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
   [junit4]    > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]    > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
   [junit4]    > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
   [junit4]    > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
   [junit4]    > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
   [junit4]    > at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_153A273D1319441C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {docs=BlockTreeOrds(blocksize=128), sparse_text=PostingsFormat(name=Memory), norms=BlockTreeOrds(blocksize=128), positions=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=BlockTreeOrds(blocksize=128), term_vectors=BlockTreeOrds(blocksize=128)}, docValues:{multi_valued_long=DocValuesFormat(name=Lucene70), foo=DocValuesFormat(name=Lucene70), numeric=DocValuesFormat(name=Memory), float=DocValuesFormat(name=Direct), multi_valued_numeric=DocValuesFormat(name=Memory), long=DocValuesFormat(name=Direct), points=DocValuesFormat(name=Asserting), sparse_int=DocValuesFormat(name=Direct), bar=DocValuesFormat(name=Memory), multi_valued_double=DocValuesFormat(name=Memory), docs=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Memory), multi_valued_int=DocValuesFormat(name=Asserting), multi_valued_bytes=DocValuesFormat(name=Memory), sparse_text=DocValuesFormat(name=Direct), double=DocValuesFormat(name=Memory), positions=DocValuesFormat(name=Lucene70), sparse_binary=DocValuesFormat(name=Direct), int=DocValuesFormat(name=Memory), sorted=DocValuesFormat(name=Memory), multi_valued_string=DocValuesFormat(name=Memory), norms=DocValuesFormat(name=Memory), sparse=DocValuesFormat(name=Direct), bytes=DocValuesFormat(name=Memory), binary=DocValuesFormat(name=Memory), dense_int=DocValuesFormat(name=Memory), multi_valued_float=DocValuesFormat(name=Lucene70), term_vectors=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1339, maxMBSortInHeap=7.94065038672138, sim=RandomSimilarity(queryNorm=true): {positions=BM25(k1=1.2,b=0.75), id=DFR GB2, term_vectors=IB SPL-D2}, locale=en-MT, timezone=Pacific/Pohnpei
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=273250320,total=436207616
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntroSorter, TestBlendedTermQuery, TestFastDecompressionMode, TestCodecUtil, TestFilterDirectoryReader, TestSortedSetSelector, TestPointQueries, TestPrefixCodedTerms, TestSumDocFreq, TestMultiTermConstantScore, TestDeterminism, TestTermQuery, LimitedFiniteStringsIteratorTest, TestRollback, TestPositiveScoresOnlyCollector, TestFieldType, TestPrefixInBooleanQuery, TestLockFactory, TestAtomicUpdate, TestCharArraySet, TestIndexOrDocValuesQuery, TestPointValues, TestIndexWriterOutOfFileDescriptors, Test2BDocs, TestSortRandom, TestFileSwitchDirectory, TestDocInverterPerFieldErrorInfo, TestMixedDocValuesUpdates, Test2BPostings, TestSearcherManager, TestReaderClosed, TestBooleanQuery, TestSparseFixedBitSet, TestNorms, TestBM25Similarity, TestReusableStringReader, TestAddIndexes, TestDocValuesQueries, TestTermdocPerf, TestPrefixRandom, TestConcurrentMergeScheduler, TestOmitTf, TestFSTs, TestMultiPhraseEnum, TestScoreCachingWrappingScorer, TestSimpleExplanationsWithFillerDocs, TestDuelingCodecsAtNight, TestBytesRefArray, TestRegExp, TestIndexFileDeleter, TestMmapDirectory, TestBytesRefAttImpl, TestFilterLeafReader, TestBlockPostingsFormat3, TestPhraseQuery, TestParallelLeafReader, TestDoubleValuesSource, TestVirtualMethod, TestDoc, TestSpanOrQuery, TestFieldInvertState, TestFieldValueQuery, TestCustomSearcherSort, TestBufferedChecksum, Test2BSortedDocValuesFixedSorted, TestRAMDirectory, TestSameScoresWithThreads, TestMatchNoDocsQuery, TestTransactions, TestLRUQueryCache, TestIndexSorting]
   [junit4] Completed [295/454 (1!)] on J1 in 182.85s, 50 tests, 1 error <<< FAILURES!

[...truncated 530 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170728_043121_1577079894741960343426.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid12707.hprof ...
   [junit4] Heap dump file created [497393830 bytes in 2.051 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 10689 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> 1814601 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/init-core-data-001
   [junit4]   2> 1814602 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1814604 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1814606 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1814606 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /xv_uud/
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1814651 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1814656 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1814658 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1814672 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36034_hdfs____3xfapk/webapp
   [junit4]   2> 1815048 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36034
   [junit4]   2> 1815128 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1815129 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1815144 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_38676_datanode____.eu3x1l/webapp
   [junit4]   2> 1815864 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38676
   [junit4]   2> 1815934 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1815935 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1815952 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_34938_datanode____.ol7j98/webapp
   [junit4]   2> 1816098 INFO  (IPC Server handler 3 on 59077) [    ] BlockStateChange BLOCK* processReport: from storage DS-617cbb16-2e7a-4317-92eb-4be5b5e09675 node DatanodeRegistration(127.0.0.1:41372, datanodeUuid=afe5a977-95c2-4f56-a6be-6985af0727ac, infoPort=36168, infoSecurePort=0, ipcPort=45814, storageInfo=lv=-56;cid=testClusterID;nsid=1091277997;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1816098 INFO  (IPC Server handler 3 on 59077) [    ] BlockStateChange BLOCK* processReport: from storage DS-beb4c5e9-14af-42f0-a5f9-bb56f2e8ac32 node DatanodeRegistration(127.0.0.1:41372, datanodeUuid=afe5a977-95c2-4f56-a6be-6985af0727ac, infoPort=36168, infoSecurePort=0, ipcPort=45814, storageInfo=lv=-56;cid=testClusterID;nsid=1091277997;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1816620 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[DCE716222F6A0505]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34938
   [junit4]   2> 1816891 INFO  (IPC Server handler 7 on 59077) [    ] BlockStateChange BLOCK* processReport: from storage DS-a3f2a88c-7c78-4dc3-b8d1-1dfe81ec0d57 node DatanodeRegistration(127.0.0.1:45700, datanodeUuid=2c1d098f-0ee2-492d-b2b5-04b873775b41, infoPort=36528, infoSecurePort=0, ipcPort=49543, storageInfo=lv=-56;cid=testClusterID;nsid=1091277997;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1816891 INFO  (IPC Server handler 7 on 59077) [    ] BlockStateChange BLOCK* processReport: from storage DS-89b61e63-d257-472e-bf87-6c29c6b0bd33 node DatanodeRegistration(127.0.0.1:45700, datanodeUuid=2c1d098f-0ee2-492d-b2b5-04b873775b41, infoPort=36528, infoSecurePort=0, ipcPort=49543, storageInfo=lv=-56;cid=testClusterID;nsid=1091277997;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1817131 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1817132 INFO  (Thread-12809) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1817132 INFO  (Thread-12809) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1817136 ERROR (Thread-12809) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1817232 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ZkTestServer start zk server on port:52619
   [junit4]   2> 1817256 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1817258 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1817260 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1817262 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1817264 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1817265 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1817267 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1817268 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1817270 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1817272 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1817273 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1817275 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1817456 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1817458 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c8fbc23{/xv_uud,null,AVAILABLE}
   [junit4]   2> 1817458 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2b69b2be{HTTP/1.1,[http/1.1]}{127.0.0.1:46862}
   [junit4]   2> 1817458 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server Started @1820398ms
   [junit4]   2> 1817461 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/xv_uud, hostPort=46862, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/control-001/cores}
   [junit4]   2> 1817462 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1817462 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1817462 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1817462 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1817462 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:18:09.221Z
   [junit4]   2> 1817466 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1817466 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/control-001/solr.xml
   [junit4]   2> 1817472 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1817476 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52619/solr
   [junit4]   2> 1817588 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1817589 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46862_xv_uud
   [junit4]   2> 1817590 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.Overseer Overseer (id=98384601998426116-127.0.0.1:46862_xv_uud-n_0000000000) starting
   [junit4]   2> 1817601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46862_xv_uud
   [junit4]   2> 1817602 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1817735 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1817741 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1817742 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1817743 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/control-001/cores
   [junit4]   2> 1817767 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1817768 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52619/solr ready
   [junit4]   2> 1817769 INFO  (qtp427977620-21396) [n:127.0.0.1:46862_xv_uud    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46862_xv_uud&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1817771 INFO  (OverseerThreadFactory-4103-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1817883 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1817883 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1817989 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1818903 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1818919 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1819014 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1819033 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1819034 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1819034 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:59077/solr_hdfs_home
   [junit4]   2> 1819034 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1819034 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1819034 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:59077/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 1819037 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 1819046 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1819046 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1819048 WARN  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
   [junit4]   2> 1824482 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1824484 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 1824514 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 1824527 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1824527 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1824539 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1824540 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=229929491, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1824586 INFO  (IPC Server handler 9 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41372 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-a3f2a88c-7c78-4dc3-b8d1-1dfe81ec0d57:NORMAL:127.0.0.1:45700|RBW], ReplicaUC[[DISK]DS-beb4c5e9-14af-42f0-a5f9-bb56f2e8ac32:NORMAL:127.0.0.1:41372|RBW]]} size 0
   [junit4]   2> 1824587 INFO  (IPC Server handler 6 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45700 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-a3f2a88c-7c78-4dc3-b8d1-1dfe81ec0d57:NORMAL:127.0.0.1:45700|RBW], ReplicaUC[[DISK]DS-beb4c5e9-14af-42f0-a5f9-bb56f2e8ac32:NORMAL:127.0.0.1:41372|RBW]]} size 0
   [junit4]   2> 1824598 WARN  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1824662 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1824662 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1824662 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 1824682 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1824682 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1824685 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=52.01171875, floorSegmentMB=0.421875, forceMergeDeletesPctAllowed=1.7159922672271488, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1824691 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2305725b[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1824693 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1824694 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1824695 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1824696 INFO  (searcherExecutor-4106-thread-1-processing-n:127.0.0.1:46862_xv_uud x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2305725b[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1824697 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153639906246656
   [junit4]   2> 1824704 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1824704 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1824704 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46862/xv_uud/control_collection_shard1_replica_n1/
   [junit4]   2> 1824704 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1824705 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46862/xv_uud/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1824705 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1824707 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1824711 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46862/xv_uud/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1824814 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1824863 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1824864 INFO  (qtp427977620-21398) [n:127.0.0.1:46862_xv_uud c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=6981
   [junit4]   2> 1824870 INFO  (qtp427977620-21396) [n:127.0.0.1:46862_xv_uud    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1824970 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1825778 INFO  (OverseerCollectionConfigSetProcessor-98384601998426116-127.0.0.1:46862_xv_uud-n_0000000000) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1825870 INFO  (qtp427977620-21396) [n:127.0.0.1:46862_xv_uud    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46862_xv_uud&wt=javabin&version=2} status=0 QTime=8101
   [junit4]   2> 1825877 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1825878 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52619/solr ready
   [junit4]   2> 1825878 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1825879 INFO  (qtp427977620-21394) [n:127.0.0.1:46862_xv_uud    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1825881 INFO  (OverseerThreadFactory-4103-thread-2-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1825882 WARN  (OverseerThreadFactory-4103-thread-2-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1826090 INFO  (qtp427977620-21394) [n:127.0.0.1:46862_xv_uud    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1826090 INFO  (qtp427977620-21394) [n:127.0.0.1:46862_xv_uud    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=210
   [junit4]   2> 1826273 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-1-001 of type NRT
   [junit4]   2> 1826274 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1826276 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@633a55c1{/xv_uud,null,AVAILABLE}
   [junit4]   2> 1826276 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@45b5cd26{HTTP/1.1,[http/1.1]}{127.0.0.1:38872}
   [junit4]   2> 1826279 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server Started @1829219ms
   [junit4]   2> 1826280 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/xv_uud, hostPort=38872, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-1-001/cores}
   [junit4]   2> 1826280 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1826280 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1826281 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1826281 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1826281 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:18:18.040Z
   [junit4]   2> 1826297 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1826297 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-1-001/solr.xml
   [junit4]   2> 1826303 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1826307 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52619/solr
   [junit4]   2> 1826321 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1826324 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1826326 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38872_xv_uud
   [junit4]   2> 1826328 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1826328 INFO  (zkCallback-1497-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1826334 INFO  (zkCallback-1502-thread-1-processing-n:127.0.0.1:38872_xv_uud) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1826487 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1826494 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1826495 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1826496 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-1-001/cores
   [junit4]   2> 1826531 INFO  (qtp2076066684-21448) [n:127.0.0.1:38872_xv_uud    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38872_xv_uud&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1826534 INFO  (OverseerThreadFactory-4103-thread-3-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38872_xv_uud for creating new replica
   [junit4]   2> 1826534 INFO  (OverseerCollectionConfigSetProcessor-98384601998426116-127.0.0.1:46862_xv_uud-n_0000000000) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1826537 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1826537 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1827558 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1827588 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
   [junit4]   2> 1827673 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1827694 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 1827695 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1827695 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:59077/solr_hdfs_home
   [junit4]   2> 1827695 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1827695 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1827695 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-1-001/cores/collection1_shard2_replica_n1], dataDir=[hdfs://localhost:59077/solr_hdfs_home/collection1/core_node2/data/]
   [junit4]   2> 1827697 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
   [junit4]   2> 1827706 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1827706 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1827710 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1827711 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node2/data
   [junit4]   2> 1827730 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node2/data/index
   [junit4]   2> 1827738 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1827738 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1827741 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1827744 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=229929491, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1827768 INFO  (IPC Server handler 6 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45700 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-617cbb16-2e7a-4317-92eb-4be5b5e09675:NORMAL:127.0.0.1:41372|RBW], ReplicaUC[[DISK]DS-89b61e63-d257-472e-bf87-6c29c6b0bd33:NORMAL:127.0.0.1:45700|RBW]]} size 0
   [junit4]   2> 1827769 INFO  (IPC Server handler 0 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41372 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-617cbb16-2e7a-4317-92eb-4be5b5e09675:NORMAL:127.0.0.1:41372|RBW], ReplicaUC[[DISK]DS-89b61e63-d257-472e-bf87-6c29c6b0bd33:NORMAL:127.0.0.1:45700|RBW]]} size 0
   [junit4]   2> 1827776 WARN  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1827854 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1827854 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1827854 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 1827867 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1827867 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1827869 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=52.01171875, floorSegmentMB=0.421875, forceMergeDeletesPctAllowed=1.7159922672271488, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1827875 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12a3637b[collection1_shard2_replica_n1] main]
   [junit4]   2> 1827876 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1827877 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1827878 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1827880 INFO  (searcherExecutor-4117-thread-1-processing-n:127.0.0.1:38872_xv_uud x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@12a3637b[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1827880 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153643243864064
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38872/xv_uud/collection1_shard2_replica_n1/
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38872/xv_uud/collection1_shard2_replica_n1/ has no replicas
   [junit4]   2> 1827888 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1827892 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38872/xv_uud/collection1_shard2_replica_n1/ shard2
   [junit4]   2> 1828043 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1828045 INFO  (qtp2076066684-21450) [n:127.0.0.1:38872_xv_uud c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1508
   [junit4]   2> 1828048 INFO  (qtp2076066684-21448) [n:127.0.0.1:38872_xv_uud    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:38872_xv_uud&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1516
   [junit4]   2> 1828207 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-2-001 of type NRT
   [junit4]   2> 1828208 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1828211 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1e0d0d9f{/xv_uud,null,AVAILABLE}
   [junit4]   2> 1828212 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@21cd0e3b{HTTP/1.1,[http/1.1]}{127.0.0.1:45892}
   [junit4]   2> 1828212 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.e.j.s.Server Started @1831152ms
   [junit4]   2> 1828212 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/xv_uud, hostPort=45892, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-2-001/cores}
   [junit4]   2> 1828212 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1828215 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1828215 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1828215 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1828215 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-28T08:18:19.974Z
   [junit4]   2> 1828221 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1828221 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-2-001/solr.xml
   [junit4]   2> 1828227 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1828231 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52619/solr
   [junit4]   2> 1828242 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1828245 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1828248 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45892_xv_uud
   [junit4]   2> 1828249 INFO  (zkCallback-1490-thread-1-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1828258 INFO  (zkCallback-1497-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1828258 INFO  (zkCallback-1502-thread-1-processing-n:127.0.0.1:38872_xv_uud) [n:127.0.0.1:38872_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1828261 INFO  (zkCallback-1508-thread-1-processing-n:127.0.0.1:45892_xv_uud) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1828363 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1828371 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1828371 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1828373 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[DCE716222F6A0505]) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-2-001/cores
   [junit4]   2> 1828412 INFO  (qtp2076066684-21449) [n:127.0.0.1:38872_xv_uud    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45892_xv_uud&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1828415 INFO  (OverseerThreadFactory-4103-thread-4-processing-n:127.0.0.1:46862_xv_uud) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:45892_xv_uud for creating new replica
   [junit4]   2> 1828415 INFO  (OverseerCollectionConfigSetProcessor-98384601998426116-127.0.0.1:46862_xv_uud-n_0000000000) [n:127.0.0.1:46862_xv_uud    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1828418 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1828418 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1829438 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1829454 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema [collection1_shard1_replica_n3] Schema name=test
   [junit4]   2> 1829555 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1829575 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n3' using configuration from collection collection1, trusted=true
   [junit4]   2> 1829575 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n3' (registry 'solr.core.collection1.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1be80524
   [junit4]   2> 1829575 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:59077/solr_hdfs_home
   [junit4]   2> 1829576 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1829576 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1829576 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [[collection1_shard1_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_DCE716222F6A0505-001/shard-2-001/cores/collection1_shard1_replica_n3], dataDir=[hdfs://localhost:59077/solr_hdfs_home/collection1/core_node4/data/]
   [junit4]   2> 1829577 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node4/data/snapshot_metadata
   [junit4]   2> 1829587 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1829587 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1829590 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1829591 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node4/data
   [junit4]   2> 1829610 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:59077/solr_hdfs_home/collection1/core_node4/data/index
   [junit4]   2> 1829617 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 1829617 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 1829620 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1829621 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=229929491, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1829634 INFO  (IPC Server handler 1 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45700 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-617cbb16-2e7a-4317-92eb-4be5b5e09675:NORMAL:127.0.0.1:41372|RBW], ReplicaUC[[DISK]DS-a3f2a88c-7c78-4dc3-b8d1-1dfe81ec0d57:NORMAL:127.0.0.1:45700|FINALIZED]]} size 0
   [junit4]   2> 1829635 INFO  (IPC Server handler 2 on 59077) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41372 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-a3f2a88c-7c78-4dc3-b8d1-1dfe81ec0d57:NORMAL:127.0.0.1:45700|FINALIZED], ReplicaUC[[DISK]DS-beb4c5e9-14af-42f0-a5f9-bb56f2e8ac32:NORMAL:127.0.0.1:41372|FINALIZED]]} size 0
   [junit4]   2> 1829644 WARN  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1829707 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1829707 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1829707 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 1829720 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1829720 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1829722 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=52.01171875, floorSegmentMB=0.421875, forceMergeDeletesPctAllowed=1.7159922672271488, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1829727 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a95726c[collection1_shard1_replica_n3] main]
   [junit4]   2> 1829728 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1829729 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1829730 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1829732 INFO  (searcherExecutor-4128-thread-1-processing-n:127.0.0.1:45892_xv_uud x:collection1_shard1_replica_n3 s:shard1 c:collection1) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [collection1_shard1_replica_n3] Registered new searcher Searcher@5a95726c[collection1_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1829732 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574153645185826816
   [junit4]   2> 1829738 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1829738 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1829738 INFO  (qtp383665616-21485) [n:127.0.0.1:45892_xv_uud c:collection1 s:shard1  

[...truncated too long message...]

ine-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=AtomicUpdateProcessorFactoryTest -Dtests.method=testMultipleThreads -Dtests.seed=DCE716222F6A0505 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=W-SU -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.05s J1 | AtomicUpdateProcessorFactoryTest.testMultipleThreads <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > at __randomizedtesting.SeedInfo.seed([DCE716222F6A0505:F0D561AFA3AE7381]:0)
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
   [junit4]    > at org.apache.solr.update.processor.AtomicUpdateProcessorFactoryTest.testMultipleThreads(AtomicUpdateProcessorFactoryTest.java:264)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > request was:q=int_i:17&wt=xml
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:879)
   [junit4]    > ... 40 more
   [junit4]   2> 4300367 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 4300367 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1321363586
   [junit4]   2> 4300367 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 4300368 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 4300370 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 4300370 INFO  (SUITE-AtomicUpdateProcessorFactoryTest-seed#[DCE716222F6A0505]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 4300370 INFO  (coreCloseExecutor-8549-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@41cefa1a
   [junit4]   2> 4300373 INFO  (coreCloseExecutor-8549-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1104083482
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.processor.AtomicUpdateProcessorFactoryTest_DCE716222F6A0505-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name_s=FSTOrd50, title_stringNoNorms=PostingsFormat(name=LuceneFixedGap), title_stemmed=BlockTreeOrds(blocksize=128), multiDefault=PostingsFormat(name=LuceneFixedGap), cat=PostingsFormat(name=LuceneFixedGap), id=FSTOrd50, text=Lucene50(blocksize=128), title=BlockTreeOrds(blocksize=128), title_lettertok=FSTOrd50}, docValues:{int_i=DocValuesFormat(name=Memory), range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Lucene70), id_i1=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Lucene70), count_i=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Asserting), range_facet_l=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1542, maxMBSortInHeap=6.1735095805599105, sim=RandomSimilarity(queryNorm=true): {}, locale=pl, timezone=W-SU
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=174853400,total=532152320
   [junit4]   2> NOTE: All tests run in this JVM: [TestSQLHandlerNonCloud, FileUtilsTest, OverseerStatusTest, HLLSerializationTest, CollectionTooManyReplicasTest, TestImpersonationWithHadoopAuth, TestDocBasedVersionConstraints, CSVRequestHandlerTest, TestElisionMultitermQuery, TestCollationFieldDocValues, ConcurrentDeleteAndCreateCollectionTest, TestManagedResourceStorage, TestConfigSets, CursorMarkTest, PKIAuthenticationIntegrationTest, SecurityConfHandlerTest, TestSolrCoreParser, SSLMigrationTest, ChaosMonkeySafeLeaderTest, TestDynamicLoading, TestBinaryResponseWriter, TestGraphTermsQParserPlugin, TestEmbeddedSolrServerSchemaAPI, TestPostingsSolrHighlighter, ShardRoutingCustomTest, TestStressRecovery, ZkStateReaderTest, ResourceLoaderTest, TestLegacyNumericRangeQueryBuilder, ReplicationFactorTest, TestSystemCollAutoCreate, DistributedQueryComponentCustomSortTest, TestManagedSchemaAPI, TestFiltering, TestStressCloudBlindAtomicUpdates, DistributedIntervalFacetingTest, ChangedSchemaMergeTest, TestNumericRangeQuery32, TestXIncludeConfig, TestUniqueKeyFieldResource, TestRTGBase, SOLR749Test, TestHttpShardHandlerFactory, TestSolrCLIRunExample, TestBM25SimilarityFactory, TlogReplayBufferedWhileIndexingTest, HdfsThreadLeakTest, TestFastLRUCache, TestSubQueryTransformerCrossCore, TermVectorComponentTest, CreateCollectionCleanupTest, TestSystemIdResolver, ShardRoutingTest, BasicFunctionalityTest, ZkSolrClientTest, RuleEngineTest, TestFreeTextSuggestions, LukeRequestHandlerTest, TestDistribDocBasedVersion, BJQParserTest, HdfsWriteToMultipleCollectionsTest, HdfsCollectionsAPIDistributedZkTest, ActionThrottleTest, SpatialFilterTest, TestRandomRequestDistribution, TestFieldCacheReopen, DistanceFunctionTest, CheckHdfsIndexTest, AnalysisAfterCoreReloadTest, TestCharFilters, TestSolrIndexConfig, DeleteLastCustomShardedReplicaTest, AssignTest, TestCopyFieldCollectionResource, TestRandomCollapseQParserPlugin, DistributedFacetPivotSmallAdvancedTest, SolrJmxReporterTest, TestUseDocValuesAsStored2, TestPerFieldSimilarity, TestInitParams, TestConfigOverlay, ZkControllerTest, IndexSchemaTest, BitVectorTest, DOMUtilTest, TestRecoveryHdfs, StatelessScriptUpdateProcessorFactoryTest, JsonLoaderTest, TestPartialUpdateDeduplication, TestHashPartitioner, CdcrBootstrapTest, TestGeoJSONResponseWriter, DistributedMLTComponentTest, TestSweetSpotSimilarityFactory, TestDistribIDF, TestManagedSynonymGraphFilterFactory, MoreLikeThisHandlerTest, HdfsLockFactoryTest, DataDrivenBlockJoinTest, TestConfigReload, CollectionsAPIDistributedZkTest, TestSlowCompositeReaderWrapper, InfoHandlerTest, TestXmlQParserPlugin, TestLegacyNumericUtils, DirectSolrSpellCheckerTest, TestSimpleQParserPlugin, TestShardHandlerFactory, SolrCmdDistributorTest, TestPerFieldSimilarityWithDefaultOverride, SharedFSAutoReplicaFailoverTest, CacheHeaderTest, AlternateDirectoryTest, NoCacheHeaderTest, MigrateRouteKeyTest, TestSolrXml, AutoScalingHandlerTest, BadComponentTest, TestJsonRequest, TestBulkSchemaConcurrent, RankQueryTest, ZkCLITest, TestDistributedSearch, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DeleteShardTest, DocValuesNotIndexedTest, ForceLeaderTest, LeaderInitiatedRecoveryOnCommitTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudRecovery, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLeaderInitiatedRecoveryThread, TestMiniSolrCloudClusterSSL, TestPullReplica, TestRandomFlRTGCloud, TestSizeLimitedDistributedMap, TestTlogReplica, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsSyncSliceTest, HdfsTlogReplayBufferedWhileIndexingTest, TestManagedStopFilterFactory, PreAnalyzedFieldManagedSchemaCloudTest, TestPointFields, TestMultiWordSynonyms, TestNoOpRegenerator, TestOverriddenPrefixQueryForCustomFieldType, TestPayloadCheckQParserPlugin, TestPayloadScoreQParserPlugin, TestQueryWrapperFilter, TestReRankQParserPlugin, TestStressUserVersions, TestTrieFacet, TestMinMaxOnMultiValuedField, BlockJoinFacetDistribTest, BlockJoinFacetRandomTest, BlockJoinFacetSimpleTest, TestNestedDocsSort, TestScoreJoinQPScore, TestLRUStatsCache, TestSha256AuthenticationProvider, ResponseHeaderTest, ConjunctionSolrSpellCheckerTest, TestAnalyzeInfixSuggestions, TestFileDictionaryLookup, TestHighFrequencyDictionaryFactory, BlockCacheTest, BlockDirectoryTest, TestFieldCacheVsDocValues, VersionInfoTest, AtomicUpdateProcessorFactoryTest]
   [junit4] Completed [704/726 (2!)] on J1 in 0.70s, 4 tests, 1 error <<< FAILURES!

[...truncated 7589 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12707.hprof

Total time: 277 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1366 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1366/

3 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([B802C60645480A62:3056F9DCEBB4679A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:     1) Thread[id=4217, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    2) Thread[id=4218, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
   1) Thread[id=4217, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   2) Thread[id=4218, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
        at __randomizedtesting.SeedInfo.seed([B802C60645480A62]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=4217, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Throwable.fillInStackTrace(Native Method)         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)         at java.lang.Throwable.<init>(Throwable.java:265)         at java.lang.Exception.<init>(Exception.java:66)         at java.lang.RuntimeException.<init>(RuntimeException.java:62)         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)         at org.apache.http.util.Asserts.check(Asserts.java:34)         at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)         at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)         at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)         at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)         at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)         at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)         at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:523)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    2) Thread[id=4218, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.security.AccessController.doPrivileged(Native Method)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)         at java.io.PrintStream.flush(PrintStream.java:338)         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at java.io.PrintStream.write(PrintStream.java:482)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)         at java.io.PrintStream.write(PrintStream.java:527)         at java.io.PrintStream.print(PrintStream.java:669)         at java.io.PrintStream.println(PrintStream.java:823)         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)         at java.lang.Throwable.printStackTrace(Throwable.java:658)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=4217, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Throwable.fillInStackTrace(Native Method)
        at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
        at java.lang.Throwable.<init>(Throwable.java:265)
        at java.lang.Exception.<init>(Exception.java:66)
        at java.lang.RuntimeException.<init>(RuntimeException.java:62)
        at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
        at org.apache.http.util.Asserts.check(Asserts.java:34)
        at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)
        at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:523)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   2) Thread[id=4218, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.security.AccessController.doPrivileged(Native Method)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
        at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.write(PrintStream.java:482)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
        at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
        at java.io.PrintStream.write(PrintStream.java:527)
        at java.io.PrintStream.print(PrintStream.java:669)
        at java.io.PrintStream.println(PrintStream.java:823)
        at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
        at java.lang.Throwable.printStackTrace(Throwable.java:658)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
        at __randomizedtesting.SeedInfo.seed([B802C60645480A62]:0)




Build Log:
[...truncated 11783 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> 961816 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[B802C60645480A62]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/init-core-data-001
   [junit4]   2> 961816 WARN  (SUITE-ChaosMonkeySafeLeaderTest-seed#[B802C60645480A62]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 961817 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[B802C60645480A62]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 961830 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[B802C60645480A62]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 961831 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[B802C60645480A62]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /l/io
   [junit4]   2> 961874 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 961875 INFO  (Thread-1005) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 961875 INFO  (Thread-1005) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 961877 ERROR (Thread-1005) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 961976 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkTestServer start zk server on port:35526
   [junit4]   2> 962021 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 962022 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 962023 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 962023 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 962024 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 962046 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 962047 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 962048 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 962048 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 962068 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 962068 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 962071 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 962386 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 962403 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d0a3516{/l/io,null,AVAILABLE}
   [junit4]   2> 962423 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5def90a8{SSL,[ssl, http/1.1]}{127.0.0.1:33282}
   [junit4]   2> 962423 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server Started @977739ms
   [junit4]   2> 962423 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/tempDir-001/control/data, hostContext=/l/io, hostPort=33282, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/control-001/cores}
   [junit4]   2> 962424 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 962424 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 962424 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 962424 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 962424 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-30T13:06:20.662Z
   [junit4]   2> 962445 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 962445 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/control-001/solr.xml
   [junit4]   2> 962449 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 962469 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35526/solr
   [junit4]   2> 962690 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 962691 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33282_l%2Fio
   [junit4]   2> 962703 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.Overseer Overseer (id=98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) starting
   [junit4]   2> 962752 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33282_l%2Fio
   [junit4]   2> 962799 INFO  (OverseerStateUpdate-98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 962984 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 962987 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 962987 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 963000 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/control-001/cores
   [junit4]   2> 963055 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 963056 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35526/solr ready
   [junit4]   2> 963794 INFO  (qtp1432794668-3711) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33282_l%252Fio&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 963810 INFO  (OverseerThreadFactory-1495-thread-1-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 964149 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 964149 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 964272 INFO  (zkCallback-460-thread-1-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 965801 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 966008 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 966617 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 966700 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 966701 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 966701 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 966701 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 966732 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=34.06640625, floorSegmentMB=1.3037109375, forceMergeDeletesPctAllowed=1.19965556417435, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6884169300874806
   [junit4]   2> 966790 WARN  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 967272 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 967272 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 967273 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 967273 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 967274 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.8525390625, noCFSRatio=0.32186447846064925]
   [junit4]   2> 967383 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@14d761e5[control_collection_shard1_replica_n1] main]
   [junit4]   2> 967386 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 967386 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 967387 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 967387 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574352962846720000
   [junit4]   2> 967399 INFO  (searcherExecutor-1498-thread-1-processing-n:127.0.0.1:33282_l%2Fio x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@14d761e5[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 967405 INFO  (zkCallback-460-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33282/l/io/control_collection_shard1_replica_n1/
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:33282/l/io/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 967418 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 967420 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33282/l/io/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 967532 INFO  (zkCallback-460-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 967533 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 967534 INFO  (qtp1432794668-3713) [n:127.0.0.1:33282_l%2Fio c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3385
   [junit4]   2> 967601 INFO  (qtp1432794668-3711) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 967635 INFO  (zkCallback-460-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 967881 INFO  (OverseerCollectionConfigSetProcessor-98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 968608 INFO  (qtp1432794668-3711) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33282_l%252Fio&wt=javabin&version=2} status=0 QTime=4814
   [junit4]   2> 968691 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 968691 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35526/solr ready
   [junit4]   2> 968691 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 968693 INFO  (qtp1432794668-3709) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=5&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 968710 INFO  (OverseerThreadFactory-1495-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 968711 WARN  (OverseerThreadFactory-1495-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 968857 INFO  (qtp1432794668-3709) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 968857 INFO  (qtp1432794668-3709) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=5&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=164
   [junit4]   2> 970024 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001 of type NRT
   [junit4]   2> 970025 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 970140 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e330220{/l/io,null,AVAILABLE}
   [junit4]   2> 970153 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6610302f{SSL,[ssl, http/1.1]}{127.0.0.1:42465}
   [junit4]   2> 970153 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server Started @985469ms
   [junit4]   2> 970153 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/l/io, hostPort=42465, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001/cores}
   [junit4]   2> 970154 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 970154 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 970154 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 970154 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 970154 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-30T13:06:28.392Z
   [junit4]   2> 970189 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 970189 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001/solr.xml
   [junit4]   2> 970222 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 970251 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35526/solr
   [junit4]   2> 970383 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 970384 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 970397 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42465_l%2Fio
   [junit4]   2> 970398 INFO  (zkCallback-467-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 970530 INFO  (zkCallback-460-thread-1-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 970676 INFO  (zkCallback-472-thread-1-processing-n:127.0.0.1:42465_l%2Fio) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 970745 INFO  (OverseerCollectionConfigSetProcessor-98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 970939 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 970969 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 970969 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 970970 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001/cores
   [junit4]   2> 971711 INFO  (qtp1896035244-3748) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42465_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 971744 INFO  (OverseerThreadFactory-1495-thread-3-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42465_l%2Fio for creating new replica
   [junit4]   2> 971879 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 971880 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 971959 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 972073 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
   [junit4]   2> 972614 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 972710 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 972711 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 972711 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 972711 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001/cores/collection1_shard2_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-1-001/cores/collection1_shard2_replica_n1/data/]
   [junit4]   2> 972742 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=34.06640625, floorSegmentMB=1.3037109375, forceMergeDeletesPctAllowed=1.19965556417435, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6884169300874806
   [junit4]   2> 972812 WARN  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 973083 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 973083 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 973084 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 973084 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 973085 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.8525390625, noCFSRatio=0.32186447846064925]
   [junit4]   2> 973086 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30740885[collection1_shard2_replica_n1] main]
   [junit4]   2> 973086 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 973087 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 973099 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 973234 INFO  (searcherExecutor-1509-thread-1-processing-n:127.0.0.1:42465_l%2Fio x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@30740885[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 973234 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574352968977743872
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42465/l/io/collection1_shard2_replica_n1/
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:42465/l/io/collection1_shard2_replica_n1/ has no replicas
   [junit4]   2> 973250 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 973253 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42465/l/io/collection1_shard2_replica_n1/ shard2
   [junit4]   2> 973428 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 973429 INFO  (qtp1896035244-3752) [n:127.0.0.1:42465_l%2Fio c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1549
   [junit4]   2> 973473 INFO  (qtp1896035244-3748) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42465_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1761
   [junit4]   2> 973753 INFO  (OverseerCollectionConfigSetProcessor-98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 975299 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001 of type NRT
   [junit4]   2> 975300 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975493 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7eaf9a8a{/l/io,null,AVAILABLE}
   [junit4]   2> 975494 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c536a39{SSL,[ssl, http/1.1]}{127.0.0.1:38417}
   [junit4]   2> 975494 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server Started @990810ms
   [junit4]   2> 975494 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/l/io, hostPort=38417, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001/cores}
   [junit4]   2> 975494 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 975543 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 975543 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975543 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 975543 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-30T13:06:33.781Z
   [junit4]   2> 975573 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 975573 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001/solr.xml
   [junit4]   2> 975667 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 975698 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35526/solr
   [junit4]   2> 975749 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 975750 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 975764 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38417_l%2Fio
   [junit4]   2> 975812 INFO  (zkCallback-467-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 975897 INFO  (zkCallback-478-thread-1-processing-n:127.0.0.1:38417_l%2Fio) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 975881 INFO  (zkCallback-472-thread-1-processing-n:127.0.0.1:42465_l%2Fio) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 975812 INFO  (zkCallback-460-thread-2-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 976341 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 976357 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 976358 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 976359 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001/cores
   [junit4]   2> 976689 INFO  (qtp1896035244-3751) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38417_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 976707 INFO  (OverseerThreadFactory-1495-thread-4-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38417_l%2Fio for creating new replica
   [junit4]   2> 976912 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n3&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 976928 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 977029 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 977128 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.s.IndexSchema [collection1_shard3_replica_n3] Schema name=test
   [junit4]   2> 977417 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 977511 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n3' using configuration from collection collection1, trusted=true
   [junit4]   2> 977511 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n3' (registry 'solr.core.collection1.shard3.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 977511 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 977511 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SolrCore [[collection1_shard3_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001/cores/collection1_shard3_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-2-001/cores/collection1_shard3_replica_n3/data/]
   [junit4]   2> 977514 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=34.06640625, floorSegmentMB=1.3037109375, forceMergeDeletesPctAllowed=1.19965556417435, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6884169300874806
   [junit4]   2> 977570 WARN  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 977746 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 977746 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 977777 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 977777 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 977778 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.8525390625, noCFSRatio=0.32186447846064925]
   [junit4]   2> 977832 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cd38512[collection1_shard3_replica_n3] main]
   [junit4]   2> 977833 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 977834 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 977844 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 977844 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574352973811679232
   [junit4]   2> 977908 INFO  (searcherExecutor-1520-thread-1-processing-n:127.0.0.1:38417_l%2Fio x:collection1_shard3_replica_n3 s:shard3 c:collection1) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SolrCore [collection1_shard3_replica_n3] Registered new searcher Searcher@6cd38512[collection1_shard3_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38417/l/io/collection1_shard3_replica_n3/
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy https://127.0.0.1:38417/l/io/collection1_shard3_replica_n3/ has no replicas
   [junit4]   2> 977944 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 977946 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38417/l/io/collection1_shard3_replica_n3/ shard3
   [junit4]   2> 978103 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 978131 INFO  (qtp1373632213-3783) [n:127.0.0.1:38417_l%2Fio c:collection1 s:shard3  x:collection1_shard3_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n3&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1218
   [junit4]   2> 978171 INFO  (qtp1896035244-3751) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:38417_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1481
   [junit4]   2> 978548 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001 of type NRT
   [junit4]   2> 978561 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 978585 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@699b76d0{/l/io,null,AVAILABLE}
   [junit4]   2> 978586 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7ffb74b4{SSL,[ssl, http/1.1]}{127.0.0.1:45494}
   [junit4]   2> 978586 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server Started @993902ms
   [junit4]   2> 978586 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/l/io, hostPort=45494, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001/cores}
   [junit4]   2> 978586 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 978604 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 978604 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 978604 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 978604 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-30T13:06:36.842Z
   [junit4]   2> 978638 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 978638 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001/solr.xml
   [junit4]   2> 978656 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 978699 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35526/solr
   [junit4]   2> 978715 INFO  (OverseerCollectionConfigSetProcessor-98397059816751109-127.0.0.1:33282_l%2Fio-n_0000000000) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 978862 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 978863 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 978864 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45494_l%2Fio
   [junit4]   2> 978881 INFO  (zkCallback-478-thread-1-processing-n:127.0.0.1:38417_l%2Fio) [n:127.0.0.1:38417_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 978967 INFO  (zkCallback-472-thread-1-processing-n:127.0.0.1:42465_l%2Fio) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 978975 INFO  (zkCallback-467-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 978975 INFO  (zkCallback-484-thread-1-processing-n:127.0.0.1:45494_l%2Fio) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 978983 INFO  (zkCallback-460-thread-1-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 979217 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 979249 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 979249 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 979251 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001/cores
   [junit4]   2> 979381 INFO  (qtp1896035244-3755) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45494_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard4&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 979402 INFO  (OverseerThreadFactory-1495-thread-5-processing-n:127.0.0.1:33282_l%2Fio) [n:127.0.0.1:33282_l%2Fio    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:45494_l%2Fio for creating new replica
   [junit4]   2> 979646 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard4_replica_n5&action=CREATE&collection=collection1&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 979646 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 980756 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 980796 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.s.IndexSchema [collection1_shard4_replica_n5] Schema name=test
   [junit4]   2> 981194 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 981242 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard4_replica_n5' using configuration from collection collection1, trusted=true
   [junit4]   2> 981243 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard4.replica_n5' (registry 'solr.core.collection1.shard4.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4
   [junit4]   2> 981243 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 981243 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SolrCore [[collection1_shard4_replica_n5] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001/cores/collection1_shard4_replica_n5], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-3-001/cores/collection1_shard4_replica_n5/data/]
   [junit4]   2> 981259 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=34.06640625, floorSegmentMB=1.3037109375, forceMergeDeletesPctAllowed=1.19965556417435, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6884169300874806
   [junit4]   2> 981275 WARN  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 981442 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 981442 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 981443 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 981443 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 981444 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.8525390625, noCFSRatio=0.32186447846064925]
   [junit4]   2> 981489 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@7cef2bfa[collection1_shard4_replica_n5] main]
   [junit4]   2> 981490 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 981490 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 981490 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 981557 INFO  (searcherExecutor-1531-thread-1-processing-n:127.0.0.1:45494_l%2Fio x:collection1_shard4_replica_n5 s:shard4 c:collection1) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SolrCore [collection1_shard4_replica_n5] Registered new searcher Searcher@7cef2bfa[collection1_shard4_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 981557 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574352977705041920
   [junit4]   2> 981586 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 981587 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 981587 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45494/l/io/collection1_shard4_replica_n5/
   [junit4]   2> 981587 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 981587 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.SyncStrategy https://127.0.0.1:45494/l/io/collection1_shard4_replica_n5/ has no replicas
   [junit4]   2> 981587 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 981588 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45494/l/io/collection1_shard4_replica_n5/ shard4
   [junit4]   2> 981697 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 981698 INFO  (qtp462202669-3813) [n:127.0.0.1:45494_l%2Fio c:collection1 s:shard4  x:collection1_shard4_replica_n5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard4_replica_n5&action=CREATE&collection=collection1&shard=shard4&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2052
   [junit4]   2> 981700 INFO  (qtp1896035244-3755) [n:127.0.0.1:42465_l%2Fio    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:45494_l%252Fio&action=ADDREPLICA&collection=collection1&shard=shard4&type=NRT&wt=javabin&version=2} status=0 QTime=2318
   [junit4]   2> 981971 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-4-001 of type NRT
   [junit4]   2> 981972 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 981997 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6eebf070{/l/io,null,AVAILABLE}
   [junit4]   2> 981997 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1e71210b{SSL,[ssl, http/1.1]}{127.0.0.1:44950}
   [junit4]   2> 981997 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.e.j.s.Server Started @997313ms
   [junit4]   2> 981997 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/l/io, hostPort=44950, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-4-001/cores}
   [junit4]   2> 981998 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 982009 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 982009 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 982009 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 982009 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-07-30T13:06:40.247Z
   [junit4]   2> 982029 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 982029 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_B802C60645480A62-001/shard-4-001/solr.xml
   [junit4]   2> 982050 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@fbb4dd4, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 982053 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[B802C60645480A62]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35526/solr
   [junit4]

[...truncated too long message...]

nes.txt -Dtests.locale=th-TH -Dtests.timezone=Pacific/Tahiti -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4]   2> REQUEST FAILED for id=0-4567
   [junit4]   2> java.lang.IllegalStateException: Connection pool shut down
   [junit4]   2> at org.apache.http.util.Asserts.check(Asserts.java:34)
   [junit4]   2> at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)
   [junit4]   2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)
   [junit4]   2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)
   [junit4]   2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:523)
   [junit4]   2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
   [junit4]   2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]   2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]   2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]   2> at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]   2> at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]   2> Jul 30, 2017 1:09:25 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[StoppableIndexingThread,5,TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> java.lang.IllegalStateException: No context information for thread: Thread[id=4217, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
   [junit4]   2> at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
   [junit4]   2> at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   [junit4]   2> Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=4217, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1848)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
   [junit4]   2> at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
   [junit4]   2>
   [junit4] ERROR   0.00s J1 | ChaosMonkeySafeLeaderTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
   [junit4]    >    1) Thread[id=4217, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   [junit4]    >    2) Thread[id=4218, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B802C60645480A62]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=4217, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]    >         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]    >         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]    >         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]    >         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]    >         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]    >         at org.apache.http.util.Asserts.check(Asserts.java:34)
   [junit4]    >         at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)
   [junit4]    >         at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)
   [junit4]    >         at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)
   [junit4]    >         at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    >         at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    >         at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    >         at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    >         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    >         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:523)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]    >         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]    >    2) Thread[id=4218, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
   [junit4]    >         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
   [junit4]    >         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
   [junit4]    >         at java.io.PrintStream.flush(PrintStream.java:338)
   [junit4]    >         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
   [junit4]    >         at java.io.PrintStream.write(PrintStream.java:482)
   [junit4]    >         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
   [junit4]    >         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
   [junit4]    >         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
   [junit4]    >         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
   [junit4]    >         at java.io.PrintStream.write(PrintStream.java:527)
   [junit4]    >         at java.io.PrintStream.print(PrintStream.java:669)
   [junit4]    >         at java.io.PrintStream.println(PrintStream.java:823)
   [junit4]    >         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:658)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:643)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:634)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B802C60645480A62]:0)
   [junit4] Completed [63/726 (1!)] on J1 in 185.25s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 45266 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1367 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1367/

28 tests failed.
FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Error from server at http://127.0.0.1:44185//collection1: Cannot use facet.mincount=0 on field a_n_tdt which is Points-based

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44185//collection1: Cannot use facet.mincount=0 on field a_n_tdt which is Points-based
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:2EE3FC596FB82ACA]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:599)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
        at org.apache.solr.TestDistributedSearch.test(TestDistributedSearch.java:241)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.TestGroupingSearch.testGroupAPI

Error Message:
Cannot use facet.mincount=0 on field foo_i which is Points-based

Stack Trace:
org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field foo_i which is Points-based
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:3B12F7CE5279B718]:0)
        at org.apache.solr.request.NumericFacets.getCountsSingleValue(NumericFacets.java:184)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:170)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:952)
        at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:921)
        at org.apache.solr.TestGroupingSearch.testGroupAPI(TestGroupingSearch.java:532)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:2EE3FC596FB82ACA]:0)
        at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1266)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudPivotFacet.test

Error Message:
init query failed: {main(facet=true&facet.pivot=pivot_tf,pivot_y_s1&facet.pivot={!stats%3Dst1}pivot_td1&facet.limit=16&facet.offset=7&facet.overrequest.count=0),extra(rows=0&q=*:*&fq=id:[*+TO+403])}: Error from server at http://127.0.0.1:39781/_/collection1: Error from server at http://127.0.0.1:39781/_/collection1_shard2_replica_n1: Cannot use facet.mincount=0 on field pivot_tf which is Points-based

Stack Trace:
java.lang.RuntimeException: init query failed: {main(facet=true&facet.pivot=pivot_tf,pivot_y_s1&facet.pivot={!stats%3Dst1}pivot_td1&facet.limit=16&facet.offset=7&facet.overrequest.count=0),extra(rows=0&q=*:*&fq=id:[*+TO+403])}: Error from server at http://127.0.0.1:39781/_/collection1: Error from server at http://127.0.0.1:39781/_/collection1_shard2_replica_n1: Cannot use facet.mincount=0 on field pivot_tf which is Points-based
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:2EE3FC596FB82ACA]:0)
        at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:274)
        at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:247)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39781/_/collection1: Error from server at http://127.0.0.1:39781/_/collection1_shard2_replica_n1: Cannot use facet.mincount=0 on field pivot_tf which is Points-based
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:267)
        ... 42 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, MMapDirectory, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, SolrIndexSearcher] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:478)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:928)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:349)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:709)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:934)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:872)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1019)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:741)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:934)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:323)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2033)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2185)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1071)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:960)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)  at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, MMapDirectory, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, SolrIndexSearcher]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:478)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:928)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:349)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:709)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:934)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:872)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1019)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:741)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:934)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:323)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2033)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2185)
        at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1071)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:960)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:981)
        at org.apache.solr.core.CoreContainer.lambda$load$11(CoreContainer.java:619)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:297)
        at sun.reflect.GeneratedMethodAccessor54.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=21711, name=searcherExecutor-7822-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
   1) Thread[id=21711, name=searcherExecutor-7822-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=21711, name=searcherExecutor-7822-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=21711, name=searcherExecutor-7822-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732]:0)


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLargeTest.test

Error Message:
Error from server at https://127.0.0.1:39908//collection1: Error from server at https://127.0.0.1:46305//collection1: Cannot use facet.mincount=0 on field hiredate_dt which is Points-based

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39908//collection1: Error from server at https://127.0.0.1:46305//collection1: Cannot use facet.mincount=0 on field hiredate_dt which is Points-based
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:2EE3FC596FB82ACA]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:557)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:605)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
        at org.apache.solr.handler.component.DistributedFacetPivotLargeTest.test(DistributedFacetPivotLargeTest.java:430)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.schema.TestPointFields.testDoublePointFieldMultiValuedFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:BA2A6FF499A9D9F3]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldMultiValuedFacetField(TestPointFields.java:2749)
        at org.apache.solr.schema.TestPointFields.testDoublePointFieldMultiValuedFacetField(TestPointFields.java:719)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_d_mv_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsMultiValued(NumericFacets.java:425)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:168)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testIntPointFieldMultiValuedFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:1876955C8BBD5720]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldMultiValuedFacetField(TestPointFields.java:2749)
        at org.apache.solr.schema.TestPointFields.testIntPointFieldMultiValuedFacetField(TestPointFields.java:321)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_i_mv_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsMultiValued(NumericFacets.java:425)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:168)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testDatePointFieldMultiValuedFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:97CAB4F7902A3E64]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldMultiValuedFacetField(TestPointFields.java:2749)
        at org.apache.solr.schema.TestPointFields.testDatePointFieldMultiValuedFacetField(TestPointFields.java:1884)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_dt_mv_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsMultiValued(NumericFacets.java:425)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:168)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testIntPointFieldFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:95B1CAF595E92EA8]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldFacetField(TestPointFields.java:2407)
        at org.apache.solr.schema.TestPointFields.testIntPointFieldFacetField(TestPointFields.java:196)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_i_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsSingleValue(NumericFacets.java:184)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:170)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testDoublePointFieldFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:240B4D43CFBF67CC]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldFacetField(TestPointFields.java:2407)
        at org.apache.solr.schema.TestPointFields.testDoublePointFieldFacetField(TestPointFields.java:597)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_d_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsSingleValue(NumericFacets.java:184)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:170)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testFloatPointFieldFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:1C0E21BCE2A8EF72]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldFacetField(TestPointFields.java:2407)
        at org.apache.solr.schema.TestPointFields.testFloatPointFieldFacetField(TestPointFields.java:1003)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_f_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsSingleValue(NumericFacets.java:184)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:170)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testLongPointFieldFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:776EC16857AF9A33]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldFacetField(TestPointFields.java:2407)
        at org.apache.solr.schema.TestPointFields.testLongPointFieldFacetField(TestPointFields.java:1358)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Cannot use facet.mincount=0 on field number_p_l_dv which is Points-based
        at org.apache.solr.request.NumericFacets.getCountsSingleValue(NumericFacets.java:184)
        at org.apache.solr.request.NumericFacets.getCounts(NumericFacets.java:170)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:499)
        at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:405)
        at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:803)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:742)
        at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:818)
        at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:329)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:273)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:860)
        ... 42 more


FAILED:  org.apache.solr.schema.TestPointFields.testDatePointFieldFacetField

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([A6B7C383C1444732:B80D7722DF6DB25B]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:846)
        at org.apache.solr.schema.TestPointFields.doTestPointFieldFacetField(TestPointFields.java:2407)
        at org.apache.solr.schema.TestPointFields.testDatePointFieldFacetField(TestPointFields.java:1700)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(Threa

[...truncated too long message...]

 params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{add=[0 (1574534191775744000), 1 (1574534191775744001), 2 (1574534191775744002), 3 (1574534191775744003), 4 (1574534191775744004), 5 (1574534191775744005), 6 (1574534191775744006), 7 (1574534191775744007), 8 (1574534191775744008), 9 (1574534191775744009)],commit=} 0 5
   [junit4]   2> 49099 INFO  (qtp124788490-476) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&fl=id,score,_docid_&wt=javabin&version=2} hits=10 status=0 QTime=0
   [junit4]   2> 49100 INFO  (qtp124788490-476) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&fl=id,score,_docid_&wt=javabin&version=2} hits=10 status=0 QTime=0
   [junit4]   2> 49100 INFO  (TEST-SolrExampleJettyTest.testStreamingRequest-seed#[72AC8F007C7B02D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStreamingRequest
   [junit4]   2> 49101 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5a761cee{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 49101 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=765784439
   [junit4]   2> 49102 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 49103 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 49104 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 49104 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5255d96
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: now flush at close
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]:   start flush: applyAllDeletes=true
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]:   index before flush _1q(8.0.0):C10
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: startFullFlush
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: coreCloseExecutor-150-thread-1 finishFullFlush success=true
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: now apply all deletes for all segments buffered updates bytesUsed=0 reader pool bytesUsed=0
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [BD][coreCloseExecutor-150-thread-1]: waitApply: no deletes to apply
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-150-thread-1]: updateMergeThreads ioThrottle=true targetMBPerSec=10240.0 MB/sec
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-150-thread-1]: now merge
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-150-thread-1]:   index: _1q(8.0.0):C10
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-150-thread-1]:   no more merges pending; now return
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: waitForMerges
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: waitForMerges done
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: start
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: enter lock
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: now prepare
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: prepareCommit: flush
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]:   index before flush _1q(8.0.0):C10
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: startFullFlush
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: now apply all deletes for all segments buffered updates bytesUsed=0 reader pool bytesUsed=0
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [BD][coreCloseExecutor-150-thread-1]: waitApply: no deletes to apply
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: coreCloseExecutor-150-thread-1 finishFullFlush success=true
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: startCommit(): start
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]:   skip startCommit(): no changes pending
   [junit4]   2> 49105 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: pendingCommit == null; skip
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: took 0.2 msec
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: commit: done
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: rollback
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: all running merges have aborted
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: rollback: done finish merges
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: abort
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-150-thread-1]: done abort success=true
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-150-thread-1]: rollback: infos=_1q(8.0.0):C10
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-150-thread-1]: now checkpoint "_1q(8.0.0):C10" [1 segments ; isCommit = false]
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-150-thread-1]: 0 msec to checkpoint
   [junit4]   2> 49106 INFO  (coreCloseExecutor-150-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=86334870
   [junit4]   2> 49110 INFO  (SUITE-SolrExampleJettyTest-seed#[72AC8F007C7B02D]-worker) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@a587bc3{/solr,null,UNAVAILABLE}
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.embedded.SolrExampleJettyTest_72AC8F007C7B02D-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=zh-CN, timezone=Etc/UCT
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=121832488,total=284164096
   [junit4]   2> NOTE: All tests run in this JVM: [TestFastInputStream, TestDocumentObjectBinder, PowerEvaluatorTest, TestUpdateRequestCodec, ClientUtilsTest, TestClusteringResponse, HyperbolicTangentEvaluatorTest, StreamExpressionParserTest, TestPathTrie, TestToleratedUpdateError, SolrExampleStreamingBinaryTest, QueryResponseTest, GetByIdTest, SolrExampleEmbeddedTest, LessThanEvaluatorTest, MergeIndexesEmbeddedTest, TangentEvaluatorTest, TestCloudSolrClientConnections, HttpSolrClientBuilderTest, SineEvaluatorTest, TestZkConfigManager, SolrExampleBinaryTest, SolrExampleXMLTest, SolrExampleJettyTest]
   [junit4] Completed [66/135 (6!)] on J1 in 5.06s, 37 tests, 1 error <<< FAILURES!

[...truncated 42603 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1368 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1368/

6 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:25DAC28276ABD120]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
.responseHeader.status!=warnings (unordered or missing)

Stack Trace:
junit.framework.AssertionFailedError: .responseHeader.status!=warnings (unordered or missing)
        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:25DAC28276ABD120]:0)
        at junit.framework.Assert.fail(Assert.java:50)
        at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
        at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:932)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:607)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
        at org.apache.solr.TestDistributedSearch.test(TestDistributedSearch.java:241)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:25DAC28276ABD120]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
        at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:160)
        at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:104)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestAuthenticationFramework.testBasics

Error Message:
Error from server at http://127.0.0.1:51381/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:51381/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:90565374E0B9E2A8]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
        at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51381/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:588)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:25DAC28276ABD120]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([AD8EFD58D857BCD8:25DAC28276ABD120]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11959 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/init-core-data-001
   [junit4]   2> 772197 WARN  (SUITE-ChaosMonkeySafeLeaderTest-seed#[AD8EFD58D857BCD8]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
   [junit4]   2> 772197 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[AD8EFD58D857BCD8]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 772198 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[AD8EFD58D857BCD8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 772199 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[AD8EFD58D857BCD8]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 772199 INFO  (SUITE-ChaosMonkeySafeLeaderTest-seed#[AD8EFD58D857BCD8]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /kx_/
   [junit4]   2> 772212 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 772212 INFO  (Thread-1586) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 772212 INFO  (Thread-1586) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 772215 ERROR (Thread-1586) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 772312 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ZkTestServer start zk server on port:60226
   [junit4]   2> 772333 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 772334 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 772336 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 772339 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 772340 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 772341 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 772342 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 772343 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 772345 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 772346 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 772347 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 772353 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 772476 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 772482 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23fa75a3{/kx_,null,AVAILABLE}
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4fb14798{SSL,[ssl, http/1.1]}{127.0.0.1:46633}
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server Started @780373ms
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/tempDir-001/control/data, hostContext=/kx_, hostPort=46633, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/control-001/cores}
   [junit4]   2> 772483 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 772483 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-02T05:30:01.574Z
   [junit4]   2> 772488 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 772488 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/control-001/solr.xml
   [junit4]   2> 772505 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 772509 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60226/solr
   [junit4]   2> 772589 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 772590 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46633_kx_
   [junit4]   2> 772590 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.c.Overseer Overseer (id=98412252450258949-127.0.0.1:46633_kx_-n_0000000000) starting
   [junit4]   2> 772595 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46633_kx_
   [junit4]   2> 772599 INFO  (zkCallback-26338-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 772648 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 772652 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 772652 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 772653 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:46633_kx_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/control-001/cores
   [junit4]   2> 772667 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 772668 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60226/solr ready
   [junit4]   2> 772691 INFO  (qtp432671262-98987) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46633_kx_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 772694 INFO  (OverseerThreadFactory-7240-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 772810 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 772811 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 772915 INFO  (zkCallback-26338-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 773823 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 773834 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 773917 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 773926 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 773926 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 773926 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 773926 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 773929 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@35acfb7d
   [junit4]   2> 773939 WARN  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 773973 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 773973 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 773974 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 773974 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 773975 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24462359847687198]
   [junit4]   2> 773976 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b852292[control_collection_shard1_replica_n1] main]
   [junit4]   2> 773977 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 773977 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 773978 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 773979 INFO  (searcherExecutor-7243-thread-1-processing-n:127.0.0.1:46633_kx_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4b852292[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 773979 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574596041055928320
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:46633/kx_/control_collection_shard1_replica_n1/
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:46633/kx_/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 773989 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 773992 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:46633/kx_/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 774094 INFO  (zkCallback-26338-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 774143 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 774146 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1336
   [junit4]   2> 774154 INFO  (qtp432671262-98987) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 774251 INFO  (zkCallback-26338-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 774696 INFO  (OverseerCollectionConfigSetProcessor-98412252450258949-127.0.0.1:46633_kx_-n_0000000000) [n:127.0.0.1:46633_kx_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 775154 INFO  (qtp432671262-98987) [n:127.0.0.1:46633_kx_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46633_kx_&wt=javabin&version=2} status=0 QTime=2463
   [junit4]   2> 775161 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 775162 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60226/solr ready
   [junit4]   2> 775162 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 775167 INFO  (qtp432671262-98982) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 775168 INFO  (OverseerThreadFactory-7240-thread-2-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 775169 WARN  (OverseerThreadFactory-7240-thread-2-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 775374 INFO  (qtp432671262-98982) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 775374 INFO  (qtp432671262-98982) [n:127.0.0.1:46633_kx_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=207
   [junit4]   2> 775473 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001 of type NRT
   [junit4]   2> 775475 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 775476 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7500a22b{/kx_,null,AVAILABLE}
   [junit4]   2> 775477 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ac47270{SSL,[ssl, http/1.1]}{127.0.0.1:48759}
   [junit4]   2> 775477 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server Started @783367ms
   [junit4]   2> 775477 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/kx_, hostPort=48759, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001/cores}
   [junit4]   2> 775479 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 775479 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 775479 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 775479 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 775479 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-02T05:30:04.570Z
   [junit4]   2> 775482 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 775482 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001/solr.xml
   [junit4]   2> 775488 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 775493 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60226/solr
   [junit4]   2> 775504 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 775506 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 775508 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48759_kx_
   [junit4]   2> 775509 INFO  (zkCallback-26338-thread-1-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 775515 INFO  (zkCallback-26345-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 775519 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 775601 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 775609 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 775609 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 775612 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:48759_kx_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001/cores
   [junit4]   2> 775635 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:48759_kx_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 775637 INFO  (OverseerCollectionConfigSetProcessor-98412252450258949-127.0.0.1:46633_kx_-n_0000000000) [n:127.0.0.1:46633_kx_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 775638 INFO  (OverseerThreadFactory-7240-thread-3-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:48759_kx_ for creating new replica
   [junit4]   2> 775674 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 775675 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 775786 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 776693 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 776705 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
   [junit4]   2> 776775 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 776785 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 776785 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 776786 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 776786 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001/cores/collection1_shard2_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-1-001/cores/collection1_shard2_replica_n1/data/]
   [junit4]   2> 776788 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@cec52e3
   [junit4]   2> 776798 WARN  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 776834 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 776834 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 776835 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 776835 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 776836 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24462359847687198]
   [junit4]   2> 776836 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47a383b1[collection1_shard2_replica_n1] main]
   [junit4]   2> 776837 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 776837 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 776838 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 776839 INFO  (searcherExecutor-7254-thread-1-processing-n:127.0.0.1:48759_kx_ x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@47a383b1[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 776839 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574596044054855680
   [junit4]   2> 776843 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 776843 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 776844 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:48759/kx_/collection1_shard2_replica_n1/
   [junit4]   2> 776844 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 776844 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:48759/kx_/collection1_shard2_replica_n1/ has no replicas
   [junit4]   2> 776844 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 776849 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:48759/kx_/collection1_shard2_replica_n1/ shard2
   [junit4]   2> 776951 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 777000 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 777001 INFO  (qtp127614899-99027) [n:127.0.0.1:48759_kx_ c:collection1 s:shard2  x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1326
   [junit4]   2> 777003 INFO  (qtp432671262-98984) [n:127.0.0.1:46633_kx_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:48759_kx_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1367
   [junit4]   2> 777102 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001 of type NRT
   [junit4]   2> 777102 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 777104 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 777110 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3def1cc7{/kx_,null,AVAILABLE}
   [junit4]   2> 777110 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@38ddcae5{SSL,[ssl, http/1.1]}{127.0.0.1:42458}
   [junit4]   2> 777110 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server Started @785001ms
   [junit4]   2> 777110 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/kx_, hostPort=42458, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001/cores}
   [junit4]   2> 777111 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 777111 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 777111 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 777111 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 777111 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-02T05:30:06.202Z
   [junit4]   2> 777117 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 777117 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001/solr.xml
   [junit4]   2> 777122 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 777124 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60226/solr
   [junit4]   2> 777134 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 777136 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 777137 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42458_kx_
   [junit4]   2> 777138 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 777138 INFO  (zkCallback-26345-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 777138 INFO  (zkCallback-26338-thread-2-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 777156 INFO  (zkCallback-26356-thread-1-processing-n:127.0.0.1:42458_kx_) [n:127.0.0.1:42458_kx_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 777205 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 777210 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 777210 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 777211 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [n:127.0.0.1:42458_kx_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001/cores
   [junit4]   2> 777276 INFO  (qtp461065706-99055) [n:127.0.0.1:42458_kx_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42458_kx_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 777283 INFO  (OverseerCollectionConfigSetProcessor-98412252450258949-127.0.0.1:46633_kx_-n_0000000000) [n:127.0.0.1:46633_kx_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 777284 INFO  (OverseerThreadFactory-7240-thread-4-processing-n:127.0.0.1:46633_kx_) [n:127.0.0.1:46633_kx_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42458_kx_ for creating new replica
   [junit4]   2> 777306 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 777306 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 777410 INFO  (zkCallback-26356-thread-1-processing-n:127.0.0.1:42458_kx_) [n:127.0.0.1:42458_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 777410 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 778321 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 778331 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema [collection1_shard1_replica_n3] Schema name=test
   [junit4]   2> 778395 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 778409 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n3' using configuration from collection collection1, trusted=true
   [junit4]   2> 778409 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n3' (registry 'solr.core.collection1.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7795d894
   [junit4]   2> 778409 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 778409 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [[collection1_shard1_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001/cores/collection1_shard1_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-2-001/cores/collection1_shard1_replica_n3/data/]
   [junit4]   2> 778412 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@598a02e0
   [junit4]   2> 778423 WARN  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 778471 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 778471 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 778472 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 778472 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 778473 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24462359847687198]
   [junit4]   2> 778473 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@49ad471[collection1_shard1_replica_n3] main]
   [junit4]   2> 778474 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 778475 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 778475 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 778475 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1574596045770326016
   [junit4]   2> 778476 INFO  (searcherExecutor-7265-thread-1-processing-n:127.0.0.1:42458_kx_ x:collection1_shard1_replica_n3 s:shard1 c:collection1) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [collection1_shard1_replica_n3] Registered new searcher Searcher@49ad471[collection1_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 778481 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 778481 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 778481 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42458/kx_/collection1_shard1_replica_n3/
   [junit4]   2> 778482 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 778482 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.SyncStrategy https://127.0.0.1:42458/kx_/collection1_shard1_replica_n3/ has no replicas
   [junit4]   2> 778482 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 778484 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42458/kx_/collection1_shard1_replica_n3/ shard1
   [junit4]   2> 778586 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 778586 INFO  (zkCallback-26356-thread-1-processing-n:127.0.0.1:42458_kx_) [n:127.0.0.1:42458_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 778634 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 778636 INFO  (qtp461065706-99057) [n:127.0.0.1:42458_kx_ c:collection1 s:shard1  x:collection1_shard1_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1330
   [junit4]   2> 778639 INFO  (qtp461065706-99055) [n:127.0.0.1:42458_kx_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42458_kx_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1362
   [junit4]   2> 778738 INFO  (zkCallback-26356-thread-1-processing-n:127.0.0.1:42458_kx_) [n:127.0.0.1:42458_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 778738 INFO  (zkCallback-26350-thread-1-processing-n:127.0.0.1:48759_kx_) [n:127.0.0.1:48759_kx_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 778740 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_AD8EFD58D857BCD8-001/shard-3-001 of type NRT
   [junit4]   2> 778742 INFO  (TEST-ChaosMonkeySafeLeaderTest.test-seed#[AD8EFD58D857BCD8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 778744 INFO  (TEST-ChaosMonkeySafeLeaderTes

[...truncated too long message...]

doop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
   [junit4]   2> at org.apache.solr.cloud.hdfs.StressHdfsTest.teardownClass(StressHdfsTest.java:78)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> ... 54 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_AD8EFD58D857BCD8-001
   [junit4]   2> Aug 02, 2017 6:20:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=sv-SE, timezone=America/Puerto_Rico
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=202881344,total=524288000
   [junit4]   2> NOTE: All tests run in this JVM: [TestMacroExpander, BlockCacheTest, IndexSchemaTest, TestClusterStateMutator, JsonLoaderTest, TestFiltering, TestPartialUpdateDeduplication, TestUpdate, CoreMergeIndexesAdminHandlerTest, SolrMetricReporterTest, ChaosMonkeySafeLeaderWithPullReplicasTest, FileUtilsTest, TestStandardQParsers, TestSolrCoreSnapshots, TestInfoStreamLogging, TestSchemaResource, HighlighterTest, TestJavabinTupleStreamParser, TestExpandComponent, TestManagedSynonymFilterFactory, TestXmlQParser, TestStressLiveNodes, SSLMigrationTest, ActionThrottleTest, DisMaxRequestHandlerTest, TestManagedSchema, TestSearcherReuse, TestDefaultStatsCache, MetricsConfigTest, DistributedExpandComponentTest, DistributedMLTComponentTest, SpatialFilterTest, DocExpirationUpdateProcessorFactoryTest, CdcrUpdateLogTest, CreateCollectionCleanupTest, TestExclusionRuleCollectionAccess, DocValuesTest, TestComplexPhraseQParserPlugin, BooleanFieldTest, DocValuesMultiTest, TestBackupRepositoryFactory, ConvertedLegacyTest, TestExactStatsCache, TestCloudRecovery, TestReloadAndDeleteDocs, TestCollectionAPI, HdfsRecoverLeaseTest, TestUnifiedSolrHighlighter, ImplicitSnitchTest, TermVectorComponentTest, CdcrBootstrapTest, OverseerStatusTest, DirectSolrConnectionTest, SolrCmdDistributorTest, DOMUtilTest, ZkSolrClientTest, TestMissingGroups, DistanceUnitsTest, PreAnalyzedUpdateProcessorTest, DistributedFacetPivotLongTailTest, CoreAdminHandlerTest, ScriptEngineTest, TestDeleteCollectionOnDownNodes, SparseHLLTest, TestOverriddenPrefixQueryForCustomFieldType, AtomicUpdateProcessorFactoryTest, TestPointFields, FullHLLTest, ReplicationFactorTest, TestEmbeddedSolrServerSchemaAPI, AssignTest, DistributedTermsComponentTest, VMParamsZkACLAndCredentialsProvidersTest, TestCloudSchemaless, TestCustomStream, TestFieldTypeCollectionResource, TestInitQParser, SmileWriterTest, TestSolr4Spatial2, TestLRUCache, TestFilteredDocIdSet, SolrIndexMetricsTest, ChaosMonkeySafeLeaderTest, TestValueSourceCache, TestSSLRandomization, TestAtomicUpdateErrorCases, TestSolrCloudWithSecureImpersonation, ConfigureRecoveryStrategyTest, TestConfigSetsAPIExclusivity, HttpSolrCallGetCoreTest, BlockJoinFacetRandomTest, ReplicaListTransformerTest, TestLeaderElectionZkExpiry, LoggingHandlerTest, TestNonDefinedSimilarityFactory, DistributedIntervalFacetingTest, TestHdfsUpdateLog, TestTrackingShardHandlerFactory, StatelessScriptUpdateProcessorFactoryTest, WrapperMergePolicyFactoryTest, AnalyticsQueryTest, TestXIncludeConfig, TestBM25SimilarityFactory, ClusterStateUpdateTest, TestFileDictionaryLookup, TestPKIAuthenticationPlugin, BlobRepositoryCloudTest, DeleteLastCustomShardedReplicaTest, TestLockTree, TestDynamicFieldResource, BadComponentTest, TestRandomFlRTGCloud, TestLegacyFieldReuse, TestPerFieldSimilarity, TestSizeLimitedDistributedMap, HdfsUnloadDistributedZkTest, CollectionTooManyReplicasTest, TestSolrIndexConfig, SharedFSAutoReplicaFailoverTest, ResourceLoaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, TestGroupingSearch, DirectUpdateHandlerTest, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTSTTest, TestCSVLoader, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, SortByFunctionTest, DistanceFunctionTest, SolrInfoBeanTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestQueryTypes, PrimitiveFieldTypeTest, TermsComponentTest, RequestHandlersTest, MoreLikeThisHandlerTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, PingRequestHandlerTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, TestBinaryField, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastOutputStream, PluginInfoTest, DateMathParserTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, CircularListTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestSimpleTrackingShardHandler, TestEmbeddedSolrServerConstructors, ConnectionReuseTest, AssignBackwardCompatibilityTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteReplicaTest, DistributedVersionInfoTest, LeaderFailoverAfterPartitionTest, MissingSegmentRecoveryTest, MoveReplicaHDFSUlogDirTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, RecoveryAfterSoftCommitTest, TestAuthenticationFramework, TestCloudPivotFacet, TestConfigSetsAPI, TestOnReconnectListenerSupport, TestPrepRecovery, TestRequestStatusCollectionAPI, TestSolrCloudWithDelegationTokens, TestStressInPlaceUpdates, HdfsBasicDistributedZk2Test, HdfsChaosMonkeySafeLeaderTest, HdfsRestartWhileUpdatingTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest]
   [junit4] Completed [573/727 (6!)] on J1 in 92.80s, 1 test, 1 failure <<< FAILURES!

[...truncated 43713 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1369 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1369/

4 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8:22E340D6C51A8400]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:     1) Thread[id=49964, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    2) Thread[id=49908, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)         at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967)         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)    3) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    7) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
   1) Thread[id=49964, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=49908, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
        at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
   3) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   7) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    5) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   5) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8]:0)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:38273/t/g/solrj_collection4, http://127.0.0.1:42885/t/g/solrj_collection4, http://127.0.0.1:51755/t/g/solrj_collection4]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:38273/t/g/solrj_collection4, http://127.0.0.1:42885/t/g/solrj_collection4, http://127.0.0.1:51755/t/g/solrj_collection4]
        at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8:22E340D6C51A8400]:0)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.queryAndAssertResultSize(SharedFSAutoReplicaFailoverTest.java:342)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:253)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51755/t/g/solrj_collection4: no servers hosting shard: shard1
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        ... 50 more




Build Log:
[...truncated 11564 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> 354018 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/init-core-data-001
   [junit4]   2> 354019 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 354019 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 354020 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 354021 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /t/g
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 354126 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 354132 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 354134 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 354152 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_38228_hdfs____.pht52b/webapp
   [junit4]   2> 354530 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38228
   [junit4]   2> 354530 WARN  (1065575191@qtp-734842590-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38228) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 354540 WARN  (1065575191@qtp-734842590-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38228) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is true
   [junit4]   2> 354746 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 354748 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 354763 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_34757_datanode____.faipz5/webapp
   [junit4]   2> 355207 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34757
   [junit4]   2> 355247 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 355248 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 355261 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48279_datanode____zbk41j/webapp
   [junit4]   2> 355526 INFO  (IPC Server handler 3 on 48578) [    ] BlockStateChange BLOCK* processReport: from storage DS-b326b4fd-1505-4e49-90b8-a76b94578322 node DatanodeRegistration(127.0.0.1:55643, datanodeUuid=8329d1c2-db40-4e36-9a28-145984f2af6b, infoPort=50167, infoSecurePort=0, ipcPort=37617, storageInfo=lv=-56;cid=testClusterID;nsid=260976441;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 355526 INFO  (IPC Server handler 3 on 48578) [    ] BlockStateChange BLOCK* processReport: from storage DS-f8a619b5-a089-454c-ac61-ad6dbc7b5f97 node DatanodeRegistration(127.0.0.1:55643, datanodeUuid=8329d1c2-db40-4e36-9a28-145984f2af6b, infoPort=50167, infoSecurePort=0, ipcPort=37617, storageInfo=lv=-56;cid=testClusterID;nsid=260976441;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 355667 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[AAB77F0C6BE6E9F8]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48279
   [junit4]   2> 355964 INFO  (IPC Server handler 4 on 48578) [    ] BlockStateChange BLOCK* processReport: from storage DS-05601783-3f4c-483f-88aa-099bfb108c7b node DatanodeRegistration(127.0.0.1:43360, datanodeUuid=6e40310f-9a0e-4eca-bebc-be3dc451f5ff, infoPort=42170, infoSecurePort=0, ipcPort=37899, storageInfo=lv=-56;cid=testClusterID;nsid=260976441;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 355964 INFO  (IPC Server handler 4 on 48578) [    ] BlockStateChange BLOCK* processReport: from storage DS-ec4c5d0f-e703-402f-ae71-b5e6583dfe56 node DatanodeRegistration(127.0.0.1:43360, datanodeUuid=6e40310f-9a0e-4eca-bebc-be3dc451f5ff, infoPort=42170, infoSecurePort=0, ipcPort=37899, storageInfo=lv=-56;cid=testClusterID;nsid=260976441;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 356053 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 356055 INFO  (Thread-615) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 356055 INFO  (Thread-615) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 356057 ERROR (Thread-615) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 356155 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ZkTestServer start zk server on port:50853
   [junit4]   2> 356195 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 356197 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 356200 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 356208 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 356209 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 356214 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 356217 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 356225 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 356227 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 356228 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 356230 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 356239 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 356436 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 356441 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7853f769{/t/g,null,AVAILABLE}
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6bd89e58{HTTP/1.1,[http/1.1]}{127.0.0.1:42885}
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server Started @361009ms
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, hostContext=/t/g, hostPort=42885, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/control-001/cores}
   [junit4]   2> 356442 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 356442 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 356443 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-07T08:14:19.108Z
   [junit4]   2> 356454 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 356454 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/control-001/solr.xml
   [junit4]   2> 356460 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 356477 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50853/solr
   [junit4]   2> 356590 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 356592 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42885_t%2Fg
   [junit4]   2> 356593 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.Overseer Overseer (id=98441210018004996-127.0.0.1:42885_t%2Fg-n_0000000000) starting
   [junit4]   2> 356601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42885_t%2Fg
   [junit4]   2> 356603 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 356713 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 356718 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 356718 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 356720 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/control-001/cores
   [junit4]   2> 356741 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 356743 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50853/solr ready
   [junit4]   2> 356745 INFO  (qtp183609570-2022) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42885_t%252Fg&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 356748 INFO  (OverseerThreadFactory-1103-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 356858 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 356859 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 356968 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 357884 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 357907 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 357996 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 358018 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 358018 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 358027 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48578/solr_hdfs_home
   [junit4]   2> 358027 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 358027 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 358027 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:48578/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 358029 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 358048 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 358048 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 358048 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
   [junit4]   2> 358703 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 358704 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 358732 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 358740 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 358740 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 358752 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 358753 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 358833 INFO  (IPC Server handler 8 on 48578) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:55643 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-05601783-3f4c-483f-88aa-099bfb108c7b:NORMAL:127.0.0.1:43360|RBW], ReplicaUC[[DISK]DS-f8a619b5-a089-454c-ac61-ad6dbc7b5f97:NORMAL:127.0.0.1:55643|FINALIZED]]} size 0
   [junit4]   2> 358835 INFO  (IPC Server handler 9 on 48578) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43360 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-05601783-3f4c-483f-88aa-099bfb108c7b:NORMAL:127.0.0.1:43360|RBW], ReplicaUC[[DISK]DS-f8a619b5-a089-454c-ac61-ad6dbc7b5f97:NORMAL:127.0.0.1:55643|FINALIZED]]} size 0
   [junit4]   2> 358845 WARN  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 358920 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 358920 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 358920 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 358937 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 358937 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 358941 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=16, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.4658203125, noCFSRatio=1.0]
   [junit4]   2> 358982 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@492a40f5[control_collection_shard1_replica_n1] main]
   [junit4]   2> 358992 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 358993 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 358997 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 358999 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575059363374039040
   [junit4]   2> 359001 INFO  (searcherExecutor-1106-thread-1-processing-n:127.0.0.1:42885_t%2Fg x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@492a40f5[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42885/t/g/control_collection_shard1_replica_n1/
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:42885/t/g/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 359009 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 359010 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 359016 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42885/t/g/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 359118 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 359168 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 359169 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2311
   [junit4]   2> 359181 INFO  (qtp183609570-2022) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 359282 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 360182 INFO  (qtp183609570-2022) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42885_t%252Fg&wt=javabin&version=2} status=0 QTime=3436
   [junit4]   2> 360206 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 360207 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50853/solr ready
   [junit4]   2> 360210 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 360212 INFO  (qtp183609570-2024) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 360214 INFO  (OverseerThreadFactory-1103-thread-2-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 360215 INFO  (OverseerCollectionConfigSetProcessor-98441210018004996-127.0.0.1:42885_t%2Fg-n_0000000000) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 360215 WARN  (OverseerThreadFactory-1103-thread-2-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 360426 INFO  (qtp183609570-2024) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 360427 INFO  (qtp183609570-2024) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=214
   [junit4]   2> 360597 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-1-001 of type NRT
   [junit4]   2> 360598 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 360600 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5124635{/t/g,null,AVAILABLE}
   [junit4]   2> 360600 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@56df1054{HTTP/1.1,[http/1.1]}{127.0.0.1:51755}
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server Started @365168ms
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/t/g, hostPort=51755, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-1-001/cores}
   [junit4]   2> 360601 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 360601 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-07T08:14:23.267Z
   [junit4]   2> 360611 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 360611 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-1-001/solr.xml
   [junit4]   2> 360617 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 360621 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50853/solr
   [junit4]   2> 360654 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 360656 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 360658 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51755_t%2Fg
   [junit4]   2> 360660 INFO  (zkCallback-280-thread-1-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 360660 INFO  (zkCallback-287-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 360665 INFO  (zkCallback-292-thread-1-processing-n:127.0.0.1:51755_t%2Fg) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 360803 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 360810 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 360810 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 360812 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-1-001/cores
   [junit4]   2> 360838 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:51755_t%252Fg&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 360841 INFO  (OverseerCollectionConfigSetProcessor-98441210018004996-127.0.0.1:42885_t%2Fg-n_0000000000) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 360844 INFO  (OverseerThreadFactory-1103-thread-3-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:51755_t%2Fg for creating new replica
   [junit4]   2> 360869 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 360870 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 360977 INFO  (zkCallback-292-thread-1-processing-n:127.0.0.1:51755_t%2Fg) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 361887 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 361902 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 361995 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 362021 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 362022 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 362022 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48578/solr_hdfs_home
   [junit4]   2> 362022 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 362022 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 362022 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[hdfs://localhost:48578/solr_hdfs_home/collection1/core_node42/data/]
   [junit4]   2> 362024 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/collection1/core_node42/data/snapshot_metadata
   [junit4]   2> 362033 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 362033 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 362042 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 362044 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/collection1/core_node42/data
   [junit4]   2> 362065 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48578/solr_hdfs_home/collection1/core_node42/data/index
   [junit4]   2> 362072 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 362072 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 362090 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 362091 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 362168 INFO  (IPC Server handler 1 on 48578) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43360 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f8a619b5-a089-454c-ac61-ad6dbc7b5f97:NORMAL:127.0.0.1:55643|RBW], ReplicaUC[[DISK]DS-ec4c5d0f-e703-402f-ae71-b5e6583dfe56:NORMAL:127.0.0.1:43360|RBW]]} size 0
   [junit4]   2> 362170 INFO  (IPC Server handler 0 on 48578) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:55643 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-ec4c5d0f-e703-402f-ae71-b5e6583dfe56:NORMAL:127.0.0.1:43360|RBW], ReplicaUC[[DISK]DS-b326b4fd-1505-4e49-90b8-a76b94578322:NORMAL:127.0.0.1:55643|FINALIZED]]} size 0
   [junit4]   2> 362184 WARN  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 362260 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 362260 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 362261 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 362274 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 362274 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 362278 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=16, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.4658203125, noCFSRatio=1.0]
   [junit4]   2> 362294 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ddf57f2[collection1_shard2_replica_n41] main]
   [junit4]   2> 362297 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 362298 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 362299 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 362300 INFO  (searcherExecutor-1117-thread-1-processing-n:127.0.0.1:51755_t%2Fg x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@5ddf57f2[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 362301 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575059366836436992
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51755/t/g/collection1_shard2_replica_n41/
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:51755/t/g/collection1_shard2_replica_n41/ has no replicas
   [junit4]   2> 362310 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 362314 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51755/t/g/collection1_shard2_replica_n41/ shard2
   [junit4]   2> 362417 INFO  (zkCallback-292-thread-1-processing-n:127.0.0.1:51755_t%2Fg) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 362466 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 362467 INFO  (qtp1451803224-2073) [n:127.0.0.1:51755_t%2Fg c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1598
   [junit4]   2> 362473 INFO  (qtp183609570-2020) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:51755_t%252Fg&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1635
   [junit4]   2> 362573 INFO  (zkCallback-292-thread-1-processing-n:127.0.0.1:51755_t%2Fg) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 362616 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-2-001 of type NRT
   [junit4]   2> 362617 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 362620 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@10132655{/t/g,null,AVAILABLE}
   [junit4]   2> 362620 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@521a2e4c{HTTP/1.1,[http/1.1]}{127.0.0.1:46852}
   [junit4]   2> 362620 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.e.j.s.Server Started @367188ms
   [junit4]   2> 362620 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/t/g, hostPort=46852, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-2-001/cores}
   [junit4]   2> 362621 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 362624 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 362625 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 362625 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 362625 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-07T08:14:25.291Z
   [junit4]   2> 362628 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 362628 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_AAB77F0C6BE6E9F8-001/shard-2-001/solr.xml
   [junit4]   2> 362635 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 362638 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50853/solr
   [junit4]   2> 362651 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:46852_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 362654 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:46852_t%2Fg    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 362657 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:46852_t%2Fg    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46852_t%2Fg
   [junit4]   2> 362658 INFO  (zkCallback-292-thread-1-processing-n:127.0.0.1:51755_t%2Fg) [n:127.0.0.1:51755_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 362659 INFO  (zkCallback-287-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 362659 INFO  (zkCallback-280-thread-2-processing-n:127.0.0.1:42885_t%2Fg) [n:127.0.0.1:42885_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 362665 INFO  (zkCallback-298-thread-1-processing-n:127.0.0.1:46852_t%2Fg) [n:127.0.0.1:46852_t%2Fg    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 362715 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[AAB77F0C6BE6E9F8]) [n:127.0.0.1:46852_t%2Fg    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2841a279
   [junit4]   2> 362722 INFO  (TEST-SharedFSAutoRe

[...truncated too long message...]

ient.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
   [junit4]    >         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
   [junit4]    >         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
   [junit4]    >         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
   [junit4]    >    3) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    7) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=49971, name=zkCallback-1615-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=49973, name=zkCallback-1615-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=49972, name=zkCallback-1615-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=49963, name=StoppableCommitThread-SendThread(127.0.0.1:47815), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    5) Thread[id=49965, name=zkCallback-1615-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([AAB77F0C6BE6E9F8]:0)
   [junit4] Completed [348/728 (2!)] on J1 in 126.77s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 44404 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1370 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1370/

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.facet.FieldFacetExtrasCloudTest

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
        at __randomizedtesting.SeedInfo.seed([21560BB5309200B3]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
        at org.apache.solr.analytics.facet.AbstractAnalyticsFacetCloudTest.setupCluster(AbstractAnalyticsFacetCloudTest.java:59)
        at org.apache.solr.analytics.facet.FieldFacetExtrasCloudTest.beforeClass(FieldFacetExtrasCloudTest.java:47)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.TestLBHttpSolrClient.testSimple

Error Message:
expected:<3> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<2>
        at __randomizedtesting.SeedInfo.seed([A4C06904A797761A:9C734DFA8064A2CB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.client.solrj.TestLBHttpSolrClient.testSimple(TestLBHttpSolrClient.java:170)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelExecutorStream

Error Message:
Error from server at https://127.0.0.1:33922/solr/workQueue_shard2_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/workQueue_shard2_replica_n3/update. Reason: <pre>    Can not find: /solr/workQueue_shard2_replica_n3/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:33922/solr/workQueue_shard2_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n3/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n3/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([A4C06904A797761A:19D71C1D9EBB4B47]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelExecutorStream(StreamExpressionTest.java:6865)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33922/solr/workQueue_shard2_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n3/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n3/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([D462F2CBF5B95C1B:5C36CD115B4531E3]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13140 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/init-core-data-001
   [junit4]   2> 3830698 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=97 numCloses=97
   [junit4]   2> 3830698 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3830700 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 3830700 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3830700 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /h_/y
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 3830754 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 3830765 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3830766 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 3830782 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_42038_hdfs____d13wu9/webapp
   [junit4]   2> 3831160 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42038
   [junit4]   2> 3831295 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3831302 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 3831318 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_34242_datanode____yqc5g6/webapp
   [junit4]   2> 3831759 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34242
   [junit4]   2> 3831876 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3831878 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 3831905 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33187_datanode____vkfxiv/webapp
   [junit4]   2> 3831990 INFO  (IPC Server handler 4 on 51901) [    ] BlockStateChange BLOCK* processReport: from storage DS-23cce0aa-59cc-44f8-83f9-e6858aed2929 node DatanodeRegistration(127.0.0.1:52882, datanodeUuid=9b732080-19d2-4915-8206-0b43867689ea, infoPort=57915, infoSecurePort=0, ipcPort=38303, storageInfo=lv=-56;cid=testClusterID;nsid=1430733612;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3831990 INFO  (IPC Server handler 4 on 51901) [    ] BlockStateChange BLOCK* processReport: from storage DS-f63c3fb5-e8ea-46dd-bbb0-5d9915c4714c node DatanodeRegistration(127.0.0.1:52882, datanodeUuid=9b732080-19d2-4915-8206-0b43867689ea, infoPort=57915, infoSecurePort=0, ipcPort=38303, storageInfo=lv=-56;cid=testClusterID;nsid=1430733612;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3832361 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[D462F2CBF5B95C1B]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33187
   [junit4]   2> 3832530 INFO  (IPC Server handler 6 on 51901) [    ] BlockStateChange BLOCK* processReport: from storage DS-d28515d8-761a-47c7-82cc-dc9049f566ee node DatanodeRegistration(127.0.0.1:41714, datanodeUuid=52812360-a8d3-4c30-be90-893f1203e62a, infoPort=36590, infoSecurePort=0, ipcPort=50497, storageInfo=lv=-56;cid=testClusterID;nsid=1430733612;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3832531 INFO  (IPC Server handler 6 on 51901) [    ] BlockStateChange BLOCK* processReport: from storage DS-bce56e04-699f-4f42-b330-62f1cb2ebca2 node DatanodeRegistration(127.0.0.1:41714, datanodeUuid=52812360-a8d3-4c30-be90-893f1203e62a, infoPort=36590, infoSecurePort=0, ipcPort=50497, storageInfo=lv=-56;cid=testClusterID;nsid=1430733612;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3833357 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3833358 INFO  (Thread-47271) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3833358 INFO  (Thread-47271) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3833362 ERROR (Thread-47271) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3833458 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.ZkTestServer start zk server on port:41097
   [junit4]   2> 3833476 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 3833478 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 3833479 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3833481 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 3833482 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 3833483 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 3833484 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 3833485 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3833487 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3833488 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 3833489 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 3833491 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 3833630 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3833632 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7dbe36a{/h_/y,null,AVAILABLE}
   [junit4]   2> 3833632 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4069e455{HTTP/1.1,[http/1.1]}{127.0.0.1:56606}
   [junit4]   2> 3833635 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.Server Started @3838348ms
   [junit4]   2> 3833635 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, hostContext=/h_/y, hostPort=56606, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/control-001/cores}
   [junit4]   2> 3833635 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3833635 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3833636 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3833636 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3833636 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-08T10:24:29.168Z
   [junit4]   2> 3833639 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3833639 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/control-001/solr.xml
   [junit4]   2> 3833644 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3833647 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41097/solr
   [junit4]   2> 3833747 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3833748 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56606_h_%2Fy
   [junit4]   2> 3833749 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.Overseer Overseer (id=98447384175640580-127.0.0.1:56606_h_%2Fy-n_0000000000) starting
   [junit4]   2> 3833759 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56606_h_%2Fy
   [junit4]   2> 3833764 INFO  (zkCallback-3098-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3833838 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3833844 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3833845 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3833846 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/control-001/cores
   [junit4]   2> 3833868 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3833869 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41097/solr ready
   [junit4]   2> 3833874 INFO  (qtp1401254264-63951) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56606_h_%252Fy&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3833878 INFO  (OverseerThreadFactory-8412-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 3833986 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3833986 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 3834092 INFO  (zkCallback-3098-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3835027 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3835042 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 3835142 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 3835158 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 3835159 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3835159 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51901/solr_hdfs_home
   [junit4]   2> 3835159 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 3835159 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3835159 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:51901/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 3835161 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51901/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 3835169 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 3835170 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 3835185 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3835186 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51901/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 3835221 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51901/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 3835229 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 3835229 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 3835246 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3835247 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=20.5888671875, floorSegmentMB=1.974609375, forceMergeDeletesPctAllowed=6.871805679395555, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.787783701248509
   [junit4]   2> 3835297 INFO  (IPC Server handler 7 on 51901) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:52882 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bce56e04-699f-4f42-b330-62f1cb2ebca2:NORMAL:127.0.0.1:41714|RBW], ReplicaUC[[DISK]DS-23cce0aa-59cc-44f8-83f9-e6858aed2929:NORMAL:127.0.0.1:52882|RBW]]} size 0
   [junit4]   2> 3835298 INFO  (IPC Server handler 9 on 51901) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41714 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-23cce0aa-59cc-44f8-83f9-e6858aed2929:NORMAL:127.0.0.1:52882|RBW], ReplicaUC[[DISK]DS-d28515d8-761a-47c7-82cc-dc9049f566ee:NORMAL:127.0.0.1:41714|FINALIZED]]} size 0
   [junit4]   2> 3835325 WARN  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 3835383 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 3835383 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3835383 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 3835394 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3835394 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3835396 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.519621200197992]
   [junit4]   2> 3835402 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@31308c19[control_collection_shard1_replica_n1] main]
   [junit4]   2> 3835403 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3835403 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3835404 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 3835406 INFO  (searcherExecutor-8415-thread-1-processing-n:127.0.0.1:56606_h_%2Fy x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@31308c19[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3835406 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575158148956684288
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56606/h_/y/control_collection_shard1_replica_n1/
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56606/h_/y/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 3835417 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 3835424 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56606/h_/y/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 3835527 INFO  (zkCallback-3098-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3835575 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3835577 INFO  (qtp1401254264-63953) [n:127.0.0.1:56606_h_%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1591
   [junit4]   2> 3835586 INFO  (qtp1401254264-63951) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 3835680 INFO  (zkCallback-3098-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3835883 INFO  (OverseerCollectionConfigSetProcessor-98447384175640580-127.0.0.1:56606_h_%2Fy-n_0000000000) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3836587 INFO  (qtp1401254264-63951) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56606_h_%252Fy&wt=javabin&version=2} status=0 QTime=2712
   [junit4]   2> 3836593 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3836594 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41097/solr ready
   [junit4]   2> 3836594 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 3836595 INFO  (qtp1401254264-63948) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3836598 INFO  (OverseerThreadFactory-8412-thread-2-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 3836599 WARN  (OverseerThreadFactory-8412-thread-2-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 3836804 INFO  (qtp1401254264-63948) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 3836804 INFO  (qtp1401254264-63948) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=209
   [junit4]   2> 3836969 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/shard-1-001 of type NRT
   [junit4]   2> 3836970 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3836971 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@793912c9{/h_/y,null,AVAILABLE}
   [junit4]   2> 3836972 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2fd786cd{HTTP/1.1,[http/1.1]}{127.0.0.1:59741}
   [junit4]   2> 3836977 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.e.j.s.Server Started @3841693ms
   [junit4]   2> 3836977 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/h_/y, hostPort=59741, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/shard-1-001/cores}
   [junit4]   2> 3836977 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3836978 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3836978 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3836978 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3836978 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-08T10:24:32.510Z
   [junit4]   2> 3836981 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3836981 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D462F2CBF5B95C1B-001/shard-1-001/solr.xml
   [junit4]   2> 3836989 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3836994 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41097/solr
   [junit4]   2> 3837006 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3837008 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3837011 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59741_h_%2Fy
   [junit4]   2> 3837012 INFO  (zkCallback-3098-thread-1-processing-n:127.0.0.1:56606_h_%2Fy) [n:127.0.0.1:56606_h_%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3837016 INFO  (zkCallback-3110-thread-1-processing-n:127.0.0.1:59741_h_%2Fy) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3837025 INFO  (zkCallback-3105-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3837114 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3837121 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[D462F2CBF5B95C1B]) [n:127.0.0.1:59741_h_%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc83b2c
   [junit4]   2> 3837121 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-see