[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 163 - Still unstable!

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
12 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 163 - Still unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/163/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudSearcherWarming.testRepFactor1LeaderStartup

Error Message:
 null Live Nodes: [127.0.0.1:54133_solr] Last available state: DocCollection(testRepFactor1LeaderStartup//collections/testRepFactor1LeaderStartup/state.json/4)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{"core_node2":{           "core":"testRepFactor1LeaderStartup_shard1_replica_n1",           "base_url":"http://127.0.0.1:54133/solr",           "node_name":"127.0.0.1:54133_solr",           "state":"active",           "type":"NRT",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError:
null
Live Nodes: [127.0.0.1:54133_solr]
Last available state: DocCollection(testRepFactor1LeaderStartup//collections/testRepFactor1LeaderStartup/state.json/4)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"testRepFactor1LeaderStartup_shard1_replica_n1",
          "base_url":"http://127.0.0.1:54133/solr",
          "node_name":"127.0.0.1:54133_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
        at __randomizedtesting.SeedInfo.seed([F3736A8DEE1E7F7A:245B2755FFFE3694]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
        at org.apache.solr.cloud.TestCloudSearcherWarming.testRepFactor1LeaderStartup(TestCloudSearcherWarming.java:126)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess

Error Message:
0

Stack Trace:
java.lang.ArrayIndexOutOfBoundsException: 0
        at __randomizedtesting.SeedInfo.seed([F3736A8DEE1E7F7A:272209C23FA04819]:0)
        at java.util.concurrent.CopyOnWriteArrayList.get(CopyOnWriteArrayList.java:387)
        at java.util.concurrent.CopyOnWriteArrayList.get(CopyOnWriteArrayList.java:396)
        at org.apache.solr.cloud.MiniSolrCloudCluster.getJettySolrRunner(MiniSolrCloudCluster.java:367)
        at org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess(TestCloudSearcherWarming.java:141)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11175 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudSearcherWarming
   [junit4]   2> 37148 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/init-core-data-001
   [junit4]   2> 37150 WARN  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 37150 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 37152 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 37154 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001
   [junit4]   2> 37154 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 37154 INFO  (Thread-20) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 37154 INFO  (Thread-20) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 37157 ERROR (Thread-20) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 37254 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:39048
   [junit4]   2> 37268 INFO  (jetty-launcher-21-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 37270 INFO  (jetty-launcher-21-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11b3c466{/solr,null,AVAILABLE}
   [junit4]   2> 37284 INFO  (jetty-launcher-21-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@f685e85{HTTP/1.1,[http/1.1]}{127.0.0.1:54133}
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.e.j.s.Server Started @41076ms
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54133}
   [junit4]   2> 37285 ERROR (jetty-launcher-21-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-09T02:35:33.120Z
   [junit4]   2> 37285 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 37290 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 37301 INFO  (jetty-launcher-21-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39048/solr
   [junit4]   2> 37539 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 37541 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54133_solr
   [junit4]   2> 37543 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.c.Overseer Overseer (id=98626734194098179-127.0.0.1:54133_solr-n_0000000000) starting
   [junit4]   2> 37616 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54133_solr
   [junit4]   2> 37625 INFO  (OverseerStateUpdate-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 37788 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54133.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3557db4e
   [junit4]   2> 37795 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54133.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3557db4e
   [junit4]   2> 37795 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54133.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3557db4e
   [junit4]   2> 37796 INFO  (jetty-launcher-21-thread-1) [n:127.0.0.1:54133_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/.
   [junit4]   2> 37833 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 37835 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39048/solr ready
   [junit4]   2> 37853 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRepFactor1LeaderStartup
   [junit4]   2> 37867 INFO  (qtp1365182105-145) [n:127.0.0.1:54133_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testRepFactor1LeaderStartup&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 37882 INFO  (OverseerThreadFactory-94-thread-1-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 37882 ERROR (OverseerThreadFactory-94-thread-1-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: testRepFactor1LeaderStartup operation: delete failed:org.apache.solr.common.SolrException: Could not find collection : testRepFactor1LeaderStartup
   [junit4]   2> at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
   [junit4]   2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.collectionCmd(OverseerCollectionMessageHandler.java:795)
   [junit4]   2> at org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:88)
   [junit4]   2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:227)
   [junit4]   2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 37906 INFO  (qtp1365182105-145) [n:127.0.0.1:54133_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testRepFactor1LeaderStartup&action=DELETE&wt=javabin&version=2} status=400 QTime=39
   [junit4]   2> 37915 INFO  (qtp1365182105-142) [n:127.0.0.1:54133_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testPeersyncFailureReplicationSuccess&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 37930 INFO  (OverseerCollectionConfigSetProcessor-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 37930 INFO  (OverseerThreadFactory-94-thread-2-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 37932 ERROR (OverseerThreadFactory-94-thread-2-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: testPeersyncFailureReplicationSuccess operation: delete failed:org.apache.solr.common.SolrException: Could not find collection : testPeersyncFailureReplicationSuccess
   [junit4]   2> at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
   [junit4]   2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.collectionCmd(OverseerCollectionMessageHandler.java:795)
   [junit4]   2> at org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:88)
   [junit4]   2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:227)
   [junit4]   2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 37938 INFO  (qtp1365182105-142) [n:127.0.0.1:54133_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testPeersyncFailureReplicationSuccess&action=DELETE&wt=javabin&version=2} status=400 QTime=23
   [junit4]   2> 37942 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&name=testRepFactor1LeaderStartup&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&createNodeSet=127.0.0.1:54133_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 37954 INFO  (OverseerCollectionConfigSetProcessor-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 37954 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.CreateCollectionCmd Create collection testRepFactor1LeaderStartup
   [junit4]   2> 37956 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.CreateCollectionCmd Only _default config set found, using it.
   [junit4]   2> 37957 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/testRepFactor1LeaderStartup/protwords.txt
   [junit4]   2> 37962 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/testRepFactor1LeaderStartup/managed-schema
   [junit4]   2> 37965 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/testRepFactor1LeaderStartup/solrconfig.xml
   [junit4]   2> 37974 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/testRepFactor1LeaderStartup/synonyms.txt
   [junit4]   2> 37978 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/testRepFactor1LeaderStartup/stopwords.txt
   [junit4]   2> 37983 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/testRepFactor1LeaderStartup/lang/contractions_it.txt
   [junit4]   2> 37989 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_no.txt
   [junit4]   2> 37994 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_hy.txt
   [junit4]   2> 37997 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/testRepFactor1LeaderStartup/lang/contractions_ca.txt
   [junit4]   2> 38000 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/testRepFactor1LeaderStartup/lang/stemdict_nl.txt
   [junit4]   2> 38004 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_id.txt
   [junit4]   2> 38007 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ga.txt
   [junit4]   2> 38010 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_da.txt
   [junit4]   2> 38014 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_hi.txt
   [junit4]   2> 38021 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ja.txt
   [junit4]   2> 38025 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_pt.txt
   [junit4]   2> 38029 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_hu.txt
   [junit4]   2> 38032 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_el.txt
   [junit4]   2> 38035 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ru.txt
   [junit4]   2> 38040 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_tr.txt
   [junit4]   2> 38044 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ar.txt
   [junit4]   2> 38049 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/testRepFactor1LeaderStartup/lang/userdict_ja.txt
   [junit4]   2> 38052 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_eu.txt
   [junit4]   2> 38057 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_it.txt
   [junit4]   2> 38061 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_cz.txt
   [junit4]   2> 38065 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/testRepFactor1LeaderStartup/lang/stoptags_ja.txt
   [junit4]   2> 38069 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/testRepFactor1LeaderStartup/lang/contractions_fr.txt
   [junit4]   2> 38072 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_de.txt
   [junit4]   2> 38076 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_fa.txt
   [junit4]   2> 38080 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/testRepFactor1LeaderStartup/lang/contractions_ga.txt
   [junit4]   2> 38083 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ca.txt
   [junit4]   2> 38087 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_nl.txt
   [junit4]   2> 38091 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_bg.txt
   [junit4]   2> 38095 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_lv.txt
   [junit4]   2> 38099 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_en.txt
   [junit4]   2> 38102 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_es.txt
   [junit4]   2> 38105 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_ro.txt
   [junit4]   2> 38109 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_th.txt
   [junit4]   2> 38112 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_fr.txt
   [junit4]   2> 38116 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_sv.txt
   [junit4]   2> 38120 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_fi.txt
   [junit4]   2> 38123 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/testRepFactor1LeaderStartup/lang/hyphenations_ga.txt
   [junit4]   2> 38126 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/testRepFactor1LeaderStartup/lang/stopwords_gl.txt
   [junit4]   2> 38130 INFO  (OverseerThreadFactory-94-thread-3-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/params.json to /configs/testRepFactor1LeaderStartup/params.json
   [junit4]   2> 38258 INFO  (OverseerStateUpdate-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testRepFactor1LeaderStartup",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testRepFactor1LeaderStartup_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:54133/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 38465 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=testRepFactor1LeaderStartup&newCollection=true&name=testRepFactor1LeaderStartup_shard1_replica_n1&action=CREATE&numShards=1&collection=testRepFactor1LeaderStartup&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 38466 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 38593 INFO  (zkCallback-28-thread-1-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 38593 INFO  (zkCallback-28-thread-2-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 39509 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.RequestParams conf resource params.json loaded . version : 0
   [junit4]   2> 39510 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 39512 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/extraction/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/extraction/lib
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-cell-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/clustering/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/clustering/lib
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-clustering-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/langid/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/langid/lib
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-langid-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 39513 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/velocity/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/velocity/lib
   [junit4]   2> 39514 WARN  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-velocity-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 39535 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 39597 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.IndexSchema [testRepFactor1LeaderStartup_shard1_replica_n1] Schema name=default-config
   [junit4]   2> 39912 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema default-config/1.6 with uniqueid field id
   [junit4]   2> 39990 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testRepFactor1LeaderStartup_shard1_replica_n1' using configuration from collection testRepFactor1LeaderStartup, trusted=true
   [junit4]   2> 39993 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54133.solr.core.testRepFactor1LeaderStartup.shard1.replica_n1' (registry 'solr.core.testRepFactor1LeaderStartup.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3557db4e
   [junit4]   2> 39998 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 39998 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [[testRepFactor1LeaderStartup_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/./testRepFactor1LeaderStartup_shard1_replica_n1/data/]
   [junit4]   2> 40029 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.XSLTResponseWriter xsltCacheLifetimeSeconds=5
   [junit4]   2> 40275 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 40275 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 40276 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 40276 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 40279 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@419529b3[testRepFactor1LeaderStartup_shard1_replica_n1] main]
   [junit4]   2> 40284 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testRepFactor1LeaderStartup
   [junit4]   2> 40284 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testRepFactor1LeaderStartup
   [junit4]   2> 40286 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/testRepFactor1LeaderStartup/managed-schema
   [junit4]   2> 40290 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 40290 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 40297 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=default,field=_text_,classname=solr.DirectSolrSpellChecker,distanceMeasure=internal,accuracy=0.5,maxEdits=2,minPrefix=1,maxInspections=5,minQueryLength=4,maxQueryFrequency=0.01}
   [junit4]   2> 40312 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 40314 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578027750416973824
   [junit4]   2> 40314 INFO  (searcherExecutor-97-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener sending requests to Searcher@419529b3[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 40314 INFO  (searcherExecutor-97-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener done.
   [junit4]   2> 40314 INFO  (searcherExecutor-97-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 40317 INFO  (searcherExecutor-97-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [testRepFactor1LeaderStartup_shard1_replica_n1] Registered new searcher Searcher@419529b3[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 40335 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 40335 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 40336 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/
   [junit4]   2> 40338 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 40339 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ has no replicas
   [junit4]   2> 40339 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 40364 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ shard1
   [junit4]   2> 40474 INFO  (zkCallback-28-thread-2-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 40474 INFO  (zkCallback-28-thread-1-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 40517 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 40520 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=testRepFactor1LeaderStartup&newCollection=true&name=testRepFactor1LeaderStartup_shard1_replica_n1&action=CREATE&numShards=1&collection=testRepFactor1LeaderStartup&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2056
   [junit4]   2> 40524 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 40629 INFO  (zkCallback-28-thread-2-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 40629 INFO  (zkCallback-28-thread-1-processing-n:127.0.0.1:54133_solr) [n:127.0.0.1:54133_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testRepFactor1LeaderStartup/state.json] for collection [testRepFactor1LeaderStartup] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 41529 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&name=testRepFactor1LeaderStartup&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&createNodeSet=127.0.0.1:54133_solr&wt=javabin&version=2} status=0 QTime=3588
   [junit4]   2> java.lang.RuntimeException
   [junit4]   2> at org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener.init(TestCloudSearcherWarming.java:306)
   [junit4]   2> at org.apache.solr.core.SolrCore.createInitInstance(SolrCore.java:826)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.verifyClass(SolrConfigHandler.java:562)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.updateNamedPlugin(SolrConfigHandler.java:533)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:482)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)
   [junit4]   2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> {event=newSearcher}
   [junit4]   2> java.lang.RuntimeException
   [junit4]   2> at org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener.init(TestCloudSearcherWarming.java:306)
   [junit4]   2> at org.apache.solr.core.SolrCore.createInitInstance(SolrCore.java:826)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.verifyClass(SolrConfigHandler.java:562)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.updateNamedPlugin(SolrConfigHandler.java:533)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:482)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)
   [junit4]   2> at org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)
   [junit4]   2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> {event=firstSearcher}
   [junit4]   2> 41563 INFO  (qtp1365182105-147) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Executed config commands successfully and persisted to ZK [{"add-listener":{
   [junit4]   2>     "name":"newSearcherListener",
   [junit4]   2>     "event":"newSearcher",
   [junit4]   2>     "class":"org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener"}}, {"add-listener":{
   [junit4]   2>     "name":"firstSearcherListener",
   [junit4]   2>     "event":"firstSearcher",
   [junit4]   2>     "class":"org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener"}}]
   [junit4]   2> 41565 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrCore config update listener called for core testRepFactor1LeaderStartup_shard1_replica_n1
   [junit4]   2> 41565 INFO  (qtp1365182105-147) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Waiting up to 30 secs for 1 replicas to set the property overlay to be of version 0 for collection testRepFactor1LeaderStartup
   [junit4]   2> 41566 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrCore core reload testRepFactor1LeaderStartup_shard1_replica_n1
   [junit4]   2> 41568 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 41582 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.RequestParams conf resource params.json loaded . version : 0
   [junit4]   2> 41583 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 41583 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/extraction/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/extraction/lib
   [junit4]   2> 41583 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-cell-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/clustering/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/clustering/lib
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-clustering-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/langid/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/langid/lib
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-langid-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/velocity/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../contrib/velocity/lib
   [junit4]   2> 41584 WARN  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist filtered by solr-velocity-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1/../../../../dist
   [junit4]   2> 41716 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 41720 INFO  (qtp1365182105-140) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 41721 INFO  (qtp1365182105-140) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.S.Request [testRepFactor1LeaderStartup_shard1_replica_n1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 41721 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 41722 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ for prop overlay   after 1 attempts
   [junit4]   2> 41722 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 41785 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.s.IndexSchema [testRepFactor1LeaderStartup_shard1_replica_n1] Schema name=default-config
   [junit4]   2> 41826 INFO  (qtp1365182105-145) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 41826 INFO  (qtp1365182105-145) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.S.Request [testRepFactor1LeaderStartup_shard1_replica_n1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 41826 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 41827 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ for prop overlay   after 2 attempts
   [junit4]   2> 41827 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 41928 INFO  (qtp1365182105-142) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 41928 INFO  (qtp1365182105-142) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.S.Request [testRepFactor1LeaderStartup_shard1_replica_n1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 41928 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 41929 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ for prop overlay   after 3 attempts
   [junit4]   2> 41929 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 41958 INFO  (OverseerCollectionConfigSetProcessor-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 42004 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.s.IndexSchema Loaded schema default-config/1.6 with uniqueid field id
   [junit4]   2> 42030 INFO  (qtp1365182105-146) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 42031 INFO  (qtp1365182105-146) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.S.Request [testRepFactor1LeaderStartup_shard1_replica_n1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 42031 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 42031 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ for prop overlay   after 4 attempts
   [junit4]   2> 42031 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 42080 INFO  (Thread-30) [n:127.0.0.1:54133_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'testRepFactor1LeaderStartup_shard1_replica_n1' using configuration from collection testRepFactor1LeaderStartup
   [junit4]   2> 42118 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54133.solr.core.testRepFactor1LeaderStartup.shard1.replica_n1' (registry 'solr.core.testRepFactor1LeaderStartup.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3557db4e
   [junit4]   2> 42118 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [[testRepFactor1LeaderStartup_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/testRepFactor1LeaderStartup_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001/tempDir-001/node1/./testRepFactor1LeaderStartup_shard1_replica_n1/data/]
   [junit4]   2> java.lang.RuntimeException
   [junit4]   2> at org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener.init(TestCloudSearcherWarming.java:306)
   [junit4]   2> at org.apache.solr.core.SolrCore.createInitInstance(SolrCore.java:826)
   [junit4]   2> at org.apache.solr.core.SolrCore.initListeners(SolrCore.java:573)
   [junit4]   2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
   [junit4]   2> at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> at org.apache.solr.core.SolrCore.lambda$getConfListener$20(SolrCore.java:2949)
   [junit4]   2> at org.apache.solr.cloud.ZkController.lambda$fireEventListeners$4(ZkController.java:2434)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> {event=newSearcher}
   [junit4]   2> java.lang.RuntimeException
   [junit4]   2> at org.apache.solr.cloud.TestCloudSearcherWarming$SleepingSolrEventListener.init(TestCloudSearcherWarming.java:306)
   [junit4]   2> at org.apache.solr.core.SolrCore.createInitInstance(SolrCore.java:826)
   [junit4]   2> at org.apache.solr.core.SolrCore.initListeners(SolrCore.java:569)
   [junit4]   2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
   [junit4]   2> at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> at org.apache.solr.core.SolrCore.lambda$getConfListener$20(SolrCore.java:2949)
   [junit4]   2> at org.apache.solr.cloud.ZkController.lambda$fireEventListeners$4(ZkController.java:2434)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> {event=firstSearcher}
   [junit4]   2> 42125 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.XSLTResponseWriter xsltCacheLifetimeSeconds=5
   [junit4]   2> 42135 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 42135 INFO  (qtp1365182105-143) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.S.Request [testRepFactor1LeaderStartup_shard1_replica_n1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 42136 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 42136 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54133/solr/testRepFactor1LeaderStartup_shard1_replica_n1/ for prop overlay   after 5 attempts
   [junit4]   2> 42136 INFO  (solrHandlerExecutor-102-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 42169 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
   [junit4]   2> 42169 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 42172 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cbdb0dc[testRepFactor1LeaderStartup_shard1_replica_n1] main]
   [junit4]   2> 42174 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testRepFactor1LeaderStartup
   [junit4]   2> 42175 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testRepFactor1LeaderStartup
   [junit4]   2> 42175 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/testRepFactor1LeaderStartup/managed-schema
   [junit4]   2> 42176 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 42176 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 42176 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=default,field=_text_,classname=solr.DirectSolrSpellChecker,distanceMeasure=internal,accuracy=0.5,maxEdits=2,minPrefix=1,maxInspections=5,minQueryLength=4,maxQueryFrequency=0.01}
   [junit4]   2> 42177 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 42178 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578027752371519488
   [junit4]   2> 42180 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener sending requests to Searcher@6cbdb0dc[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 42180 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener done.
   [junit4]   2> 42180 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 42182 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 42199 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [testRepFactor1LeaderStartup_shard1_replica_n1] Registered new searcher Searcher@6cbdb0dc[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 42199 INFO  (Thread-30) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@41aa6a97[testRepFactor1LeaderStartup_shard1_replica_n1] main]
   [junit4]   2> 42200 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener sending requests to Searcher@41aa6a97[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 42200 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.QuerySenderListener QuerySenderListener done.
   [junit4]   2> 42205 INFO  (searcherExecutor-103-thread-1-processing-n:127.0.0.1:54133_solr x:testRepFactor1LeaderStartup_shard1_replica_n1 s:shard1 c:testRepFactor1LeaderStartup r:core_node2) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [testRepFactor1LeaderStartup_shard1_replica_n1] Registered new searcher Searcher@41aa6a97[testRepFactor1LeaderStartup_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 42238 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler I already have the expected version 0 of config
   [junit4]   2> 42238 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.h.SolrConfigHandler isStale false , resourceloader org.apache.solr.cloud.ZkSolrResourceLoader
   [junit4]   2> 42238 INFO  (qtp1365182105-144) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node

[...truncated too long message...]

 42328 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 42328 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@281cb791: rootName = solr_54133, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@32b5c9f2
   [junit4]   2> 42332 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 42332 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7aa6caeb: rootName = solr_54133, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@73a5f7bd
   [junit4]   2> 42335 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 42335 INFO  (coreCloseExecutor-107-thread-1) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.c.SolrCore [testRepFactor1LeaderStartup_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@41f7a253
   [junit4]   2> 42335 INFO  (coreCloseExecutor-107-thread-1) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testRepFactor1LeaderStartup.shard1.replica_n1, tag=1106747987
   [junit4]   2> 42335 INFO  (coreCloseExecutor-107-thread-1) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@8ebcd6d: rootName = solr_54133, domain = solr.core.testRepFactor1LeaderStartup.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testRepFactor1LeaderStartup.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@14c25c29
   [junit4]   2> 42345 INFO  (coreCloseExecutor-107-thread-1) [n:127.0.0.1:54133_solr c:testRepFactor1LeaderStartup s:shard1 r:core_node2 x:testRepFactor1LeaderStartup_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testRepFactor1LeaderStartup.shard1.leader, tag=1106747987
   [junit4]   2> 42349 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.c.Overseer Overseer (id=98626734194098179-127.0.0.1:54133_solr-n_0000000000) closing
   [junit4]   2> 42350 INFO  (OverseerStateUpdate-98626734194098179-127.0.0.1:54133_solr-n_0000000000) [n:127.0.0.1:54133_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54133_solr
   [junit4]   2> 42355 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@11b3c466{/solr,null,UNAVAILABLE}
   [junit4]   2> 42357 INFO  (zkCallback-33-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 132357 INFO  (TEST-TestCloudSearcherWarming.testRepFactor1LeaderStartup-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testRepFactor1LeaderStartup
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudSearcherWarming -Dtests.method=testRepFactor1LeaderStartup -Dtests.seed=F3736A8DEE1E7F7A -Dtests.slow=true -Dtests.locale=es-PE -Dtests.timezone=Africa/Nairobi -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 94.5s J0 | TestCloudSearcherWarming.testRepFactor1LeaderStartup <<<
   [junit4]    > Throwable #1: java.lang.AssertionError:
   [junit4]    > null
   [junit4]    > Live Nodes: [127.0.0.1:54133_solr]
   [junit4]    > Last available state: DocCollection(testRepFactor1LeaderStartup//collections/testRepFactor1LeaderStartup/state.json/4)={
   [junit4]    >   "pullReplicas":"0",
   [junit4]    >   "replicationFactor":"1",
   [junit4]    >   "shards":{"shard1":{
   [junit4]    >       "range":"80000000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{"core_node2":{
   [junit4]    >           "core":"testRepFactor1LeaderStartup_shard1_replica_n1",
   [junit4]    >           "base_url":"http://127.0.0.1:54133/solr",
   [junit4]    >           "node_name":"127.0.0.1:54133_solr",
   [junit4]    >           "state":"active",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "leader":"true"}}}},
   [junit4]    >   "router":{"name":"compositeId"},
   [junit4]    >   "maxShardsPerNode":"1",
   [junit4]    >   "autoAddReplicas":"false",
   [junit4]    >   "nrtReplicas":"1",
   [junit4]    >   "tlogReplicas":"0"}
   [junit4]    > at __randomizedtesting.SeedInfo.seed([F3736A8DEE1E7F7A:245B2755FFFE3694]:0)
   [junit4]    > at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
   [junit4]    > at org.apache.solr.cloud.TestCloudSearcherWarming.testRepFactor1LeaderStartup(TestCloudSearcherWarming.java:126)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 132367 INFO  (TEST-TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testPeersyncFailureReplicationSuccess
   [junit4]   2> 132367 INFO  (TEST-TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess-seed#[F3736A8DEE1E7F7A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testPeersyncFailureReplicationSuccess
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudSearcherWarming -Dtests.method=testPeersyncFailureReplicationSuccess -Dtests.seed=F3736A8DEE1E7F7A -Dtests.slow=true -Dtests.locale=es-PE -Dtests.timezone=Africa/Nairobi -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.01s J0 | TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess <<<
   [junit4]    > Throwable #1: java.lang.ArrayIndexOutOfBoundsException: 0
   [junit4]    > at __randomizedtesting.SeedInfo.seed([F3736A8DEE1E7F7A:272209C23FA04819]:0)
   [junit4]    > at java.util.concurrent.CopyOnWriteArrayList.get(CopyOnWriteArrayList.java:387)
   [junit4]    > at java.util.concurrent.CopyOnWriteArrayList.get(CopyOnWriteArrayList.java:396)
   [junit4]    > at org.apache.solr.cloud.MiniSolrCloudCluster.getJettySolrRunner(MiniSolrCloudCluster.java:367)
   [junit4]    > at org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess(TestCloudSearcherWarming.java:141)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 132376 ERROR (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 132376 INFO  (SUITE-TestCloudSearcherWarming-seed#[F3736A8DEE1E7F7A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39048 39048
   [junit4]   2> 137795 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 137797 INFO  (Thread-20) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39048 39048
   [junit4]   2> 137805 WARN  (Thread-20) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/aliases.json
   [junit4]   2> 2 /solr/configs/testRepFactor1LeaderStartup/managed-schema
   [junit4]   2> 2 /solr/clusterprops.json
   [junit4]   2> 2 /solr/configs/testRepFactor1LeaderStartup
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 7 /solr/collections/testRepFactor1LeaderStartup/state.json
   [junit4]   2> 2 /solr/clusterstate.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/live_nodes
   [junit4]   2> 2 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudSearcherWarming_F3736A8DEE1E7F7A-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1343, maxMBSortInHeap=6.181701377200373, sim=RandomSimilarity(queryNorm=true): {}, locale=es-PE, timezone=Africa/Nairobi
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=73278920,total=130240512
   [junit4]   2> NOTE: All tests run in this JVM: [TestNumericTokenStream, TestUnifiedSolrHighlighter, BJQParserTest, UniqFieldsUpdateProcessorFactoryTest, TestLegacyNumericRangeQueryBuilder, TestSort, HLLSerializationTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSQLHandlerNonCloud, TestSearcherReuse, AlternateDirectoryTest, TestCloudSearcherWarming]
   [junit4] Completed [19/731 (1!)] on J0 in 100.68s, 2 tests, 1 failure, 1 error <<< FAILURES!

[...truncated 48675 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 164 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/164/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([592CCE92A469C34E:DA5A91607210CDEF]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:184)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=27257, name=jetty-launcher-5906-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=27253, name=jetty-launcher-5906-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
   1) Thread[id=27257, name=jetty-launcher-5906-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=27253, name=jetty-launcher-5906-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
        at __randomizedtesting.SeedInfo.seed([592CCE92A469C34E]:0)




Build Log:
[...truncated 12976 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
   [junit4]   2> 2578853 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/init-core-data-001
   [junit4]   2> 2578853 WARN  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 2578854 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2578855 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2578855 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001
   [junit4]   2> 2578855 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2578856 INFO  (Thread-5667) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2578856 INFO  (Thread-5667) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2578858 ERROR (Thread-5667) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2578956 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:37312
   [junit4]   2> 2578970 INFO  (jetty-launcher-3342-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2578971 INFO  (jetty-launcher-3342-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2578974 INFO  (jetty-launcher-3342-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@290579a8{/solr,null,AVAILABLE}
   [junit4]   2> 2578976 INFO  (jetty-launcher-3342-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f3d7217{HTTP/1.1,[http/1.1]}{127.0.0.1:48759}
   [junit4]   2> 2578977 INFO  (jetty-launcher-3342-thread-1) [    ] o.e.j.s.Server Started @2582539ms
   [junit4]   2> 2578977 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48759}
   [junit4]   2> 2578978 ERROR (jetty-launcher-3342-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2578978 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2578978 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2578978 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2578978 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-09T11:09:23.926Z
   [junit4]   2> 2578979 INFO  (jetty-launcher-3342-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d76cfd1{/solr,null,AVAILABLE}
   [junit4]   2> 2578979 INFO  (jetty-launcher-3342-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1f119700{HTTP/1.1,[http/1.1]}{127.0.0.1:39994}
   [junit4]   2> 2578979 INFO  (jetty-launcher-3342-thread-2) [    ] o.e.j.s.Server Started @2582542ms
   [junit4]   2> 2578979 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39994}
   [junit4]   2> 2578980 ERROR (jetty-launcher-3342-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2578980 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2578980 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2578980 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2578980 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-09T11:09:23.928Z
   [junit4]   2> 2578985 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2578986 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2579000 INFO  (jetty-launcher-3342-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37312/solr
   [junit4]   2> 2579000 INFO  (jetty-launcher-3342-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37312/solr
   [junit4]   2> 2579125 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2579126 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39994_solr
   [junit4]   2> 2579128 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.c.Overseer Overseer (id=98628754722324485-127.0.0.1:39994_solr-n_0000000000) starting
   [junit4]   2> 2579134 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2579135 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48759_solr
   [junit4]   2> 2579137 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2579137 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2579138 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39994_solr
   [junit4]   2> 2579139 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2579141 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2579236 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48759.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579243 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48759.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579243 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48759.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579244 INFO  (jetty-launcher-3342-thread-1) [n:127.0.0.1:48759_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node1/.
   [junit4]   2> 2579258 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39994.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579262 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39994.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579263 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39994.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2579263 INFO  (jetty-launcher-3342-thread-2) [n:127.0.0.1:39994_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node2/.
   [junit4]   2> 2579282 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2579283 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37312/solr ready
   [junit4]   2> 2579288 INFO  (qtp1320178805-21515) [n:127.0.0.1:39994_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2579291 INFO  (OverseerThreadFactory-8920-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2579408 INFO  (OverseerStateUpdate-98628754722324485-127.0.0.1:39994_solr-n_0000000000) [n:127.0.0.1:39994_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:48759/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 2579414 INFO  (OverseerStateUpdate-98628754722324485-127.0.0.1:39994_solr-n_0000000000) [n:127.0.0.1:39994_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:39994/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 2579423 INFO  (OverseerStateUpdate-98628754722324485-127.0.0.1:39994_solr-n_0000000000) [n:127.0.0.1:39994_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:48759/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 2579431 INFO  (OverseerStateUpdate-98628754722324485-127.0.0.1:39994_solr-n_0000000000) [n:127.0.0.1:39994_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:39994/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 2579637 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2579638 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2579638 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2579639 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2579639 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2579639 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2579762 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2579762 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2579763 INFO  (zkCallback-3354-thread-2-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2579763 INFO  (zkCallback-3353-thread-2-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2580684 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2580685 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2580686 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2580694 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2580694 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema [collection1_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 2580696 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema [collection1_shard2_replica_t4] Schema name=minimal
   [junit4]   2> 2580694 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema [collection1_shard2_replica_t6] Schema name=minimal
   [junit4]   2> 2580704 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2580704 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t2' using configuration from collection collection1, trusted=true
   [junit4]   2> 2580704 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39994.solr.core.collection1.shard1.replica_t2' (registry 'solr.core.collection1.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2580704 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2580705 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [[collection1_shard1_replica_t2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node2/collection1_shard1_replica_t2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node2/./collection1_shard1_replica_t2/data/]
   [junit4]   2> 2580705 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2580705 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t6' using configuration from collection collection1, trusted=true
   [junit4]   2> 2580705 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39994.solr.core.collection1.shard2.replica_t6' (registry 'solr.core.collection1.shard2.replica_t6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2580705 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2580705 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [[collection1_shard2_replica_t6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node2/collection1_shard2_replica_t6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node2/./collection1_shard2_replica_t6/data/]
   [junit4]   2> 2580707 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2580707 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t4' using configuration from collection collection1, trusted=true
   [junit4]   2> 2580707 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48759.solr.core.collection1.shard2.replica_t4' (registry 'solr.core.collection1.shard2.replica_t4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2580707 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2580707 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [[collection1_shard2_replica_t4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node1/collection1_shard2_replica_t4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node1/./collection1_shard2_replica_t4/data/]
   [junit4]   2> 2580711 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 2580712 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2580712 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
   [junit4]   2> 2580713 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48759.solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5917cb21
   [junit4]   2> 2580713 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2580713 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [[collection1_shard1_replica_t1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node1/collection1_shard1_replica_t1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_592CCE92A469C34E-001/tempDir-001/node1/./collection1_shard1_replica_t1/data/]
   [junit4]   2> 2580764 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2580764 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2580765 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2580765 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2580766 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6df3d113[collection1_shard1_replica_t1] main]
   [junit4]   2> 2580767 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2580768 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2580768 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2580768 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2580768 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2580769 INFO  (searcherExecutor-8928-thread-1-processing-n:127.0.0.1:48759_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@6df3d113[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2580769 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578060077562068992
   [junit4]   2> 2580769 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2580769 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2580770 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.SolrIndexSearcher Opening [Searcher@77142d1d[collection1_shard2_replica_t4] main]
   [junit4]   2> 2580772 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2580772 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2580773 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2580773 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578060077566263296
   [junit4]   2> 2580773 INFO  (searcherExecutor-8927-thread-1-processing-n:127.0.0.1:48759_solr x:collection1_shard2_replica_t4 s:shard2 c:collection1 r:core_node7) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [collection1_shard2_replica_t4] Registered new searcher Searcher@77142d1d[collection1_shard2_replica_t4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2580774 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2580774 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2580775 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2580775 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2580776 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 2580777 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d2af98a[collection1_shard2_replica_t6] main]
   [junit4]   2> 2580779 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2580779 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2580779 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 2580780 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2580780 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578060077573603328
   [junit4]   2> 2580781 INFO  (searcherExecutor-8926-thread-1-processing-n:127.0.0.1:39994_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@1d2af98a[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2580792 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2580792 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2580793 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2580793 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2580794 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@43bb126f[collection1_shard1_replica_t2] main]
   [junit4]   2> 2580795 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2580796 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2580796 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2580796 INFO  (searcherExecutor-8925-thread-1-processing-n:127.0.0.1:39994_solr x:collection1_shard1_replica_t2 s:shard1 c:collection1 r:core_node5) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [collection1_shard1_replica_t2] Registered new searcher Searcher@43bb126f[collection1_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2580797 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578060077591429120
   [junit4]   2> 2580885 INFO  (zkCallback-3353-thread-2-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2580885 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2580885 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2580885 INFO  (zkCallback-3354-thread-2-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581277 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2581278 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2581278 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48759/solr/collection1_shard1_replica_t1/
   [junit4]   2> 2581278 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t1 url=http://127.0.0.1:48759/solr START replicas=[http://127.0.0.1:39994/solr/collection1_shard1_replica_t2/] nUpdates=100
   [junit4]   2> 2581280 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2581281 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2581281 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/
   [junit4]   2> 2581282 INFO  (qtp1320178805-21519) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2581282 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=http://127.0.0.1:48759/solr START replicas=[http://127.0.0.1:39994/solr/collection1_shard2_replica_t6/] nUpdates=100
   [junit4]   2> 2581284 INFO  (qtp1320178805-21509) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2581580 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t1 url=http://127.0.0.1:48759/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 2581580 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2581580 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2581580 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2581581 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ZkController collection1_shard1_replica_t1 stopping background replication from leader
   [junit4]   2> 2581584 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=http://127.0.0.1:48759/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 2581584 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2581584 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2581584 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2581585 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController collection1_shard2_replica_t4 stopping background replication from leader
   [junit4]   2> 2581592 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48759/solr/collection1_shard1_replica_t1/ shard1
   [junit4]   2> 2581594 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/ shard2
   [junit4]   2> 2581703 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581703 INFO  (zkCallback-3354-thread-2-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581703 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581703 INFO  (zkCallback-3353-thread-2-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581748 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2581748 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2581751 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2113
   [junit4]   2> 2581752 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2115
   [junit4]   2> 2581818 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ZkController collection1_shard2_replica_t6 starting background replication from leader
   [junit4]   2> 2581819 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2581818 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ZkController collection1_shard1_replica_t2 starting background replication from leader
   [junit4]   2> 2581819 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2581821 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 2581821 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 2581822 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication
   [junit4]   2> 2581823 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
   [junit4]   2> 2581826 INFO  (qtp1320178805-21517) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2188
   [junit4]   2> 2581830 INFO  (qtp1320178805-21518) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2191
   [junit4]   2> 2581834 INFO  (qtp1320178805-21515) [n:127.0.0.1:39994_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2581934 INFO  (zkCallback-3353-thread-2-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581934 INFO  (zkCallback-3353-thread-1-processing-n:127.0.0.1:39994_solr) [n:127.0.0.1:39994_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581934 INFO  (zkCallback-3354-thread-2-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2581934 INFO  (zkCallback-3354-thread-1-processing-n:127.0.0.1:48759_solr) [n:127.0.0.1:48759_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2582840 INFO  (qtp1320178805-21515) [n:127.0.0.1:39994_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=3552
   [junit4]   2> 2582840 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 2582841 INFO  (SUITE-TestCloudRecovery-seed#[592CCE92A469C34E]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 2582882 INFO  (TEST-TestCloudRecovery.leaderRecoverFromLogOnStartupTest-seed#[592CCE92A469C34E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting leaderRecoverFromLogOnStartupTest
   [junit4]   2> 2582891 INFO  (qtp1320178805-21516) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1578060079784001536&distrib.from=http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/&wt=javabin&version=2}{deleteByQuery=*:* (-1578060079784001536)} 0 0
   [junit4]   2> 2582891 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:48759/solr/collection1_shard1_replica_t1/&wt=javabin&version=2}{deleteByQuery=*:* (-1578060079784001536)} 0 4
   [junit4]   2> 2582892 INFO  (qtp1320178805-21520) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1578060079781904384&distrib.from=http://127.0.0.1:48759/solr/collection1_shard1_replica_t1/&wt=javabin&version=2}{deleteByQuery=*:* (-1578060079781904384)} 0 1
   [junit4]   2> 2582892 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1578060079781904384)} 0 7
   [junit4]   2> 2582894 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578060079790292992,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2582894 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2582895 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578060079791341568,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2582895 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2582895 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2582895 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2582895 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 2582895 INFO  (qtp1320178805-21516) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 2582895 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
   [junit4]   2> 2582896 INFO  (qtp1320178805-21509) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 2582898 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2582898 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2583297 INFO  (OverseerCollectionConfigSetProcessor-98628754722324485-127.0.0.1:39994_solr-n_0000000000) [n:127.0.0.1:39994_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2583403 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2583403 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2583908 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2583908 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2584412 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2584414 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2584821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
   [junit4]   2> 2584821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:48759/solr/collection1_shard2_replica_t4/
   [junit4]   2> 2584821 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
   [junit4]   2> 2584822 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:48759/solr/collection1_shard1_replica_t1/
   [junit4]   2> 2584832 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2584832 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2584832 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 2584833 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2584833 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 2584833 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 2584833 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 2584834 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2584833 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2584834 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@52c4c4a9 commitCommandVersion:0
   [junit4]   2> 2584834 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@5288d3f commitCommandVersion:0
   [junit4]   2> 2584839 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@13f9ce92[collection1_shard2_replica_t6] main]
   [junit4]   2> 2584839 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@44158704[collection1_shard1_replica_t2] main]
   [junit4]   2> 2584840 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2584840 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2584842 INFO  (searcherExecutor-8926-thread-1-processing-n:127.0.0.1:39994_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@13f9ce92[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2584842 INFO  (searcherExecutor-8925-thread-1-processing-n:127.0.0.1:39994_solr x:collection1_shard1_replica_t2 s:shard1 c:collection1 r:core_node5) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [collection1_shard1_replica_t2] Registered new searcher Searcher@44158704[collection1_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2584917 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2584918 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=1
   [junit4]   2> 2585422 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2585422 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2585926 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2585927 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2586431 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2586431 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2586934 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2586934 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2587439 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2587439 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2587820 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2587820 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2587821 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2587821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2587821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2587821 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2587821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2587821 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2587821 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 1504955369782
   [junit4]   2> 2587821 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 1504955369782
   [junit4]   2> 2587944 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2587944 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2588447 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2588447 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2588951 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2588951 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2589455 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2589461 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2589963 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2589965 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2590470 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2590470 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2590827 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2590827 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2590827 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2590827 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2590827 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 2590828 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 2590828 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2590827 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2590828 INFO  (indexFetcher-8948-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 1504955369782
   [junit4]   2> 2590828 INFO  (indexFetcher-8947-thread-1) [n:127.0.0.1:39994_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 1504955369782
   [junit4]   2> 2590982 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2590982 INFO  (qtp2007135903-21508) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2591486 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2591490 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2591994 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2591996 INFO  (qtp2007135903-21507) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2592500 INFO  (qtp2007135903-21504) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2592510 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2593012 INFO  (qtp2007135903-21510) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2593014 INFO  (qtp2007135903-21506) [n:127.0.0.1:48759_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2593518 INFO  (qtp2007135903-21511) [n:127.0.0.1:48759_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.

[...truncated too long message...]

estManagedSchemaThreadSafety, DistribDocExpirationUpdateProcessorTest, TestRequestForwarding, DistributedQueryComponentCustomSortTest, PreAnalyzedUpdateProcessorTest, ConnectionManagerTest, TestLuceneMatchVersion, TermVectorComponentDistributedTest, SpellPossibilityIteratorTest, CachingDirectoryFactoryTest, TestSystemCollAutoCreate, TestSearchPerf, BasicAuthStandaloneTest, TestSimpleQParserPlugin, AtomicUpdatesTest, TestLRUCache, TestSortByMinMaxFunction, HLLUtilTest, TestConfig, TestCollationField, LukeRequestHandlerTest, TestStressUserVersions, TestCollectionsAPIViaSolrCloudCluster, ShufflingReplicaListTransformerTest, TestWordDelimiterFilterFactory, TestLockTree, TestJoin, ConnectionReuseTest, TestSubQueryTransformerCrossCore, BigEndianAscendingWordDeserializerTest, TolerantUpdateProcessorTest, TestTrackingShardHandlerFactory, LeaderFailoverAfterPartitionTest, TestEmbeddedSolrServerSchemaAPI, BadComponentTest, TriLevelCompositeIdRoutingTest, TestDocumentBuilder, TestSQLHandler, TestDistributedGrouping, CoreSorterTest, SuggesterTSTTest, SortSpecParsingTest, DistributedDebugComponentTest, BasicFunctionalityTest, TestMultiWordSynonyms, SynonymTokenizerTest, TestXmlQParser, CloneFieldUpdateProcessorFactoryTest, TestDynamicFieldResource, TestComplexPhraseQParserPlugin, TestFileDictionaryLookup, TestChildDocTransformer, SolrCoreCheckLockOnStartupTest, SolrMetricManagerTest, LeaderElectionIntegrationTest, SpellingQueryConverterTest, SaslZkACLProviderTest, TestTlogReplica, TestFieldResource, TestFieldCacheReopen, DistributedMLTComponentTest, TestPullReplica, TestSolrCloudWithDelegationTokens, DataDrivenBlockJoinTest, MoveReplicaTest, TestSuggestSpellingConverter, TestCloudManagedSchema, TestCharFilters, TestLMJelinekMercerSimilarityFactory, MoveReplicaHDFSTest, HdfsSyncSliceTest, OverseerModifyCollectionTest, MoreLikeThisHandlerTest, CoreAdminRequestStatusTest, ExternalFileFieldSortTest, TestSystemIdResolver, ClassificationUpdateProcessorIntegrationTest, TestMergePolicyConfig, TestGraphMLResponseWriter, HdfsNNFailoverTest, TestCSVLoader, CloudExitableDirectoryReaderTest, TestRandomFlRTGCloud, TestOverriddenPrefixQueryForCustomFieldType, TestMacroExpander, BasicZkTest, SuggesterFSTTest, OpenExchangeRatesOrgProviderTest, ChaosMonkeySafeLeaderTest, TestSimpleTrackingShardHandler, TemplateUpdateProcessorTest, TestSolrIndexConfig, ZkStateWriterTest, TestNumericRangeQuery64, FullHLLTest, TestHdfsBackupRestoreCore, SpellCheckCollatorTest, RemoteQueryErrorTest, TestClusterProperties, TestFastWriter, TestInfoStreamLogging, DistributedFacetPivotSmallTest, QueryEqualityTest, SolrIndexConfigTest, TestReRankQParserPlugin, TestManagedResource, TestZkChroot, TestOnReconnectListenerSupport, TestTrie, TestDistribDocBasedVersion, TestObjectReleaseTracker, PreAnalyzedFieldManagedSchemaCloudTest, SolrGraphiteReporterTest, TestUniqueKeyFieldResource, SimpleMLTQParserTest, TestBadConfig, DistributedSuggestComponentTest, HdfsChaosMonkeySafeLeaderTest, TestFieldCacheWithThreads, TestConfigSetsAPIExclusivity, AssignTest, TestFastOutputStream, BlockJoinFacetSimpleTest, TestAuthenticationFramework, SolrIndexMetricsTest, BadCopyFieldTest, DeleteLastCustomShardedReplicaTest, MigrateRouteKeyTest, FileUtilsTest, TestRealTimeGet, TestClusterStateMutator, BlockJoinFacetDistribTest, TestComponentsName, TestRestManager, TestMultiValuedNumericRangeQuery, ShardRoutingCustomTest, TestConfigOverlay, ChangedSchemaMergeTest, AliasIntegrationTest, ConcurrentDeleteAndCreateCollectionTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, LeaderElectionTest, DirectUpdateHandlerTest, TestIndexSearcher, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, PolyFieldTest, SchemaVersionSpecificBehaviorTest, FieldMutatingUpdateProcessorTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, SolrInfoBeanTest, IndexBasedSpellCheckerTest, TestQueryUtils, DirectSolrSpellCheckerTest, TestQueryTypes, FileBasedSpellCheckerTest, RequestHandlersTest, DocumentBuilderTest, TestValueSourceCache, TermVectorComponentTest, FastVectorHighlighterTest, RegexBoostProcessorTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, CSVRequestHandlerTest, SearchHandlerTest, MultiTermTest, SampleTest, TestDocSet, MinimalSchemaTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, TestCodecSupport, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, URLClassifyProcessorTest, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedMissingSort, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CleanupOldIndexTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DistributedQueueTest, DocValuesNotIndexedTest, ForceLeaderTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncReplicationTest, ShardSplitTest, TestCollectionAPI, TestConfigSetsAPI, TestPrepRecovery, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=592CCE92A469C34E -Dtests.slow=true -Dtests.locale=mt -Dtests.timezone=EAT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
   [junit4]    >    1) Thread[id=27257, name=jetty-launcher-5906-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=27253, name=jetty-launcher-5906-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([592CCE92A469C34E]:0)
   [junit4] Completed [545/731 (2!)] on J1 in 58.09s, 12 tests, 1 error <<< FAILURES!

[...truncated 46838 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 165 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/165/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.update.AutoCommitTest.testMaxTime

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([68B30D02F50A0768:F24770E06B909B54]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
        at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
        xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

        request was:q=id:529&qt=&start=0&rows=20&version=2.2
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
        ... 40 more




Build Log:
[...truncated 11188 lines...]
   [junit4] Suite: org.apache.solr.update.AutoCommitTest
   [junit4]   2> 102073 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001/init-core-data-001
   [junit4]   2> 102073 WARN  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 102074 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 102075 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 102076 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 102077 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 102108 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 102129 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 102539 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 102588 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 102592 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 102593 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 102594 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 102626 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 102643 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 102726 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 102743 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 102743 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 102743 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 102743 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001/init-core-data-001/]
   [junit4]   2> 102744 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1566622038, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 102807 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 102807 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 102811 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 102811 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 102812 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=44, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15270368466321715]
   [junit4]   2> 102813 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@652adadc[collection1] main]
   [junit4]   2> 102814 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 102818 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 102846 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 102846 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 102847 INFO  (coreLoadExecutor-443-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578086137384140800
   [junit4]   2> 102847 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 102847 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 102847 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 102847 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 102847 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 102848 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 102848 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 102848 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 102849 INFO  (searcherExecutor-444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@652adadc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 102851 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 102852 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMaxTime
   [junit4]   2> 102853 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 102854 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e087f3f[collection1] realtime]
   [junit4]   2> 102855 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 2
   [junit4]   2> 102855 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties
   [junit4]   2> 102856 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 102888 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 102906 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 103012 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 103023 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 103057 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 103057 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001/init-core-data-001/]
   [junit4]   2> 103120 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 103120 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 103121 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@10411ec2[collection1] main]
   [junit4]   2> 103124 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 103125 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 103129 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 103130 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578086137680887808
   [junit4]   2> 103134 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 103135 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=578665496, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 103136 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 103136 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 103136 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 103137 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 103137 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 103138 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 103138 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 103139 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 103139 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 103141 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@10411ec2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 103141 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66f80e7f[collection1] main]
   [junit4]   2> 103141 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1e212070
   [junit4]   2> 103141 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=505487472
   [junit4]   2> 103142 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6fb6da50: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@4fd38bc1
   [junit4]   2> 103151 INFO  (searcherExecutor-449-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@66f80e7f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 103153 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1578086137700810752)]} 0 4
   [junit4]   2> 103154 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 103163 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1578086137715490816)]} 0 14
   [junit4]   2> 103163 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 103164 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2>
   [junit4]   2> request was:q=id:529&qt=&start=0&rows=20&version=2.2
   [junit4]   2> 103165 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=id:529&qt=&start=0&rows=20&version=2.2:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2>
   [junit4]   2> request was:q=id:529&qt=&start=0&rows=20&version=2.2
   [junit4]   2> at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
   [junit4]   2> at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 103166 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=AutoCommitTest -Dtests.method=testMaxTime -Dtests.seed=68B30D02F50A0768 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.32s J1 | AutoCommitTest.testMaxTime <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > at __randomizedtesting.SeedInfo.seed([68B30D02F50A0768:F24770E06B909B54]:0)
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
   [junit4]    > at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > request was:q=id:529&qt=&start=0&rows=20&version=2.2
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
   [junit4]    > ... 40 more
   [junit4]   2> 103171 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
   [junit4]   2> 103172 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 103172 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6366ff51[collection1] realtime]
   [junit4]   2> 103173 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 1
   [junit4]   2> 103173 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties
   [junit4]   2> 103175 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 103200 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 103233 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 103337 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 103348 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 103396 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 103396 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001/init-core-data-001/]
   [junit4]   2> 103449 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 103449 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 103450 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b96be87[collection1] main]
   [junit4]   2> 103450 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 103451 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 103454 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 103455 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 103455 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=7.751953125, floorSegmentMB=1.0849609375, forceMergeDeletesPctAllowed=27.743052247476868, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8921464933741545
   [junit4]   2> 103456 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 103456 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 103456 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 103456 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 103456 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 103456 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 103457 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 103457 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 103457 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 103459 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3b96be87[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 103459 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5b0fffd[collection1] main]
   [junit4]   2> 103460 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@12aa36f6
   [junit4]   2> 103460 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=313145078
   [junit4]   2> 103460 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@499c8b3a: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@4fd38bc1
   [junit4]   2> 103462 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5b0fffd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 103467 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1578086138034257920)]} 0 0
   [junit4]   2> 103468 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 107468 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 107480 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f28c099[collection1] main]
   [junit4]   2> 107480 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 107482 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3f28c099[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.1.0):C1)))}
   [junit4]   2> 107483 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1578086142245339136)]} 0 4018
   [junit4]   2> 107484 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 107485 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[529 (-1578086142247436288)]} 0 4019
   [junit4]   2> 107485 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 109485 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 109488 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@520c5248[collection1] main]
   [junit4]   2> 109488 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 109490 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@520c5248[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(7.1.0):C1)))}
   [junit4]   2> 109493 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 109493 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 109494 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144354025472)]} 0 6030
   [junit4]   2> 109495 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144354025473)]} 0 6030
   [junit4]   2> 109495 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144355074048)]} 0 6030
   [junit4]   2> 109495 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144355074049)]} 0 6031
   [junit4]   2> 109495 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144355074050)]} 0 6031
   [junit4]   2> 109496 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144356122624)]} 0 6031
   [junit4]   2> 109496 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144356122625)]} 0 6031
   [junit4]   2> 109496 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144356122626)]} 0 6031
   [junit4]   2> 109496 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144356122627)]} 0 6031
   [junit4]   2> 109496 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1578086144356122628)]} 0 6032
   [junit4]   2> 109497 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 109497 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144357171200)]} 0 6032
   [junit4]   2> 109497 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144357171201)]} 0 6032
   [junit4]   2> 109497 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144357171202)]} 0 6033
   [junit4]   2> 109497 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144357171203)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144357171204)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144358219776)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144358219777)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144358219778)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144358219779)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1578086144358219780)]} 0 6033
   [junit4]   2> 109498 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 110498 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 110501 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16587756[collection1] main]
   [junit4]   2> 110501 INFO  (commitScheduler-456-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 110503 INFO  (searcherExecutor-453-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@16587756[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.1.0):C10/9:delGen=1)))}
   [junit4]   2> 110504 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 110505 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 110505 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testCommitWithin
   [junit4]   2> 110507 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testMaxDocs
   [junit4]   2> 110507 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 110509 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d971bf3[collection1] realtime]
   [junit4]   2> 110510 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 2
   [junit4]   2> 110510 ERROR (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties
   [junit4]   2> 110510 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 110546 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 110578 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 110658 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 110666 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 110688 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f2bcfc2
   [junit4]   2> 110688 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001/init-core-data-001/]
   [junit4]   2> 110744 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 110744 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 110745 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ebcde35[collection1] main]
   [junit4]   2> 110746 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 110746 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 110749 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 110749 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 110749 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 110749 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 110749 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 110750 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 110750 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 110751 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 110751 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 110751 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 110752 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=32.4580078125, floorSegmentMB=2.1103515625, forceMergeDeletesPctAllowed=13.981081599167142, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 110752 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 110759 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4ebcde35[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 110759 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2a071707[collection1] main]
   [junit4]   2> 110760 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@75a336c1
   [junit4]   2> 110760 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1973630657
   [junit4]   2> 110760 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@587c6c16: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@4fd38bc1
   [junit4]   2> 110761 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2a071707[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 110774 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[0 (1578086145695154176)]} 0 1
   [junit4]   2> 110774 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[1 (1578086145696202752)]} 0 1
   [junit4]   2> 110775 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[2 (1578086145696202753)]} 0 1
   [junit4]   2> 110775 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[3 (1578086145697251328)]} 0 2
   [junit4]   2> 110775 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[4 (1578086145697251329)]} 0 2
   [junit4]   2> 110778 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[5 (1578086145697251330)]} 0 5
   [junit4]   2> 110779 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[6 (1578086145701445632)]} 0 6
   [junit4]   2> 110779 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[7 (1578086145701445633)]} 0 6
   [junit4]   2> 110780 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[8 (1578086145701445634)]} 0 6
   [junit4]   2> 110780 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[9 (1578086145702494208)]} 0 6
   [junit4]   2> 110780 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[10 (1578086145702494209)]} 0 7
   [junit4]   2> 110781 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[11 (1578086145702494210)]} 0 7
   [junit4]   2> 110781 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[12 (1578086145703542784)]} 0 8
   [junit4]   2> 110781 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[13 (1578086145703542785)]} 0 8
   [junit4]   2> 110781 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 110782 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[14 (1578086145704591360)]} 0 9
   [junit4]   2> 110783 INFO  (commitScheduler-460-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 110788 INFO  (commitScheduler-460-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57de44d3[collection1] main]
   [junit4]   2> 110788 INFO  (commitScheduler-460-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 110791 INFO  (searcherExecutor-457-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@57de44d3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.1.0):C15)))}
   [junit4]   2> 111034 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[15 (1578086145967783936)]} 0 261
   [junit4]   2> 111034 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:14&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 111035 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:15&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 111035 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[68B30D02F50A0768]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxDocs
   [junit4]   2> 111036 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 111036 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1963230992
   [junit4]   2> 111036 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 111036 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3723eb65: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@64b45517
   [junit4]   2> 111039 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 111039 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@e02c391: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4d39ab1b
   [junit4]   2> 111042 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 111042 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@472a55bc: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6518e8ed
   [junit4]   2> 111043 INFO  (SUITE-AutoCommitTest-seed#[68B30D02F50A0768]-worker) [    x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 111043 INFO  (coreCloseExecutor-461-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1c2deefe
   [junit4]   2> 111043 INFO  (coreCloseExecutor-461-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=472772350
   [junit4]   2> 111043 INFO  (coreCloseExecutor-461-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6826bc77: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@4fd38bc1
   [junit4]   2> 111053 INFO  (coreCloseExecutor-461-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 111053 INFO  (coreCloseExecutor-461-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@18af7f72 commitCommandVersion:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_68B30D02F50A0768-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field_t=PostingsFormat(name=Memory), multiDefault=FST50, subject=FST50, id=PostingsFormat(name=Memory), text=FSTOrd50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Memory), id_i1=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1595, maxMBSortInHeap=7.4865426892854465, sim=RandomSimilarity(queryNorm=true): {}, locale=es-AR, timezone=Africa/Porto-Novo
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=68823888,total=136167424
   [junit4]   2> NOTE: All tests run in this JVM: [UpdateLogTest, TestGraphTermsQParserPlugin, ConvertedLegacyTest, BadIndexSchemaTest, TestCloudInspectUtil, TestAuthorizationFramework, DeleteLastCustomShardedReplicaTest, TestDistribDocBasedVersion, QueryResultKeyTest, NodeMutatorTest, MetricUtilsTest, ConcurrentDeleteAndCreateCollectionTest, TestComplexPhraseQParserPlugin, TestSolrCLIRunExample, ChangedSchemaMergeTest, AutoCommitTest]
   [junit4] Completed [25/731 (1!)] on J1 in 9.00s, 3 tests, 1 error <<< FAILURES!

[...truncated 48635 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 166 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/166/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestPullReplica.testKillLeader

Error Message:
Replica state not updated in cluster state null Live Nodes: [127.0.0.1:56987_solr, 127.0.0.1:52489_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={   "pullReplicas":"1",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"pull_replica_test_kill_leader_shard1_replica_n1",           "base_url":"http://127.0.0.1:56987/solr",           "node_name":"127.0.0.1:56987_solr",           "state":"down",           "type":"NRT",           "leader":"true"},         "core_node4":{           "core":"pull_replica_test_kill_leader_shard1_replica_p2",           "base_url":"http://127.0.0.1:52489/solr",           "node_name":"127.0.0.1:52489_solr",           "state":"active",           "type":"PULL"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"100",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Replica state not updated in cluster state
null
Live Nodes: [127.0.0.1:56987_solr, 127.0.0.1:52489_solr]
Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={
  "pullReplicas":"1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"pull_replica_test_kill_leader_shard1_replica_n1",
          "base_url":"http://127.0.0.1:56987/solr",
          "node_name":"127.0.0.1:56987_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node4":{
          "core":"pull_replica_test_kill_leader_shard1_replica_p2",
          "base_url":"http://127.0.0.1:52489/solr",
          "node_name":"127.0.0.1:52489_solr",
          "state":"active",
          "type":"PULL"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"100",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
        at __randomizedtesting.SeedInfo.seed([F0CB252CA943B7CC:B9DDD198CBF8239A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
        at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401)
        at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11978 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestPullReplica
   [junit4]   2> 1341528 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/init-core-data-001
   [junit4]   2> 1341529 WARN  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 1341529 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1341530 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1341530 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001
   [junit4]   2> 1341531 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1341531 INFO  (Thread-3169) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1341531 INFO  (Thread-3169) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1341568 ERROR (Thread-3169) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1341632 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:49361
   [junit4]   2> 1341646 INFO  (jetty-launcher-2850-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1341646 INFO  (jetty-launcher-2850-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1341648 INFO  (jetty-launcher-2850-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c7266be{/solr,null,AVAILABLE}
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e0e0f96{/solr,null,AVAILABLE}
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2ca6d05d{HTTP/1.1,[http/1.1]}{127.0.0.1:56987}
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-1) [    ] o.e.j.s.Server Started @1344821ms
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56987}
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6286992b{HTTP/1.1,[http/1.1]}{127.0.0.1:52489}
   [junit4]   2> 1341649 ERROR (jetty-launcher-2850-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-2) [    ] o.e.j.s.Server Started @1344821ms
   [junit4]   2> 1341649 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52489}
   [junit4]   2> 1341652 ERROR (jetty-launcher-2850-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T02:30:43.731Z
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1341652 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T02:30:43.731Z
   [junit4]   2> 1341656 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1341656 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1341664 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e699ff29b0001, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1341664 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e699ff29b0002, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1341665 INFO  (jetty-launcher-2850-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49361/solr
   [junit4]   2> 1341666 INFO  (jetty-launcher-2850-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49361/solr
   [junit4]   2> 1341819 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1341823 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52489_solr
   [junit4]   2> 1341824 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.c.Overseer Overseer (id=98632377539690501-127.0.0.1:52489_solr-n_0000000000) starting
   [junit4]   2> 1341824 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1341826 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56987_solr
   [junit4]   2> 1341835 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1341841 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1341846 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52489_solr
   [junit4]   2> 1341847 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1341847 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1341912 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1341919 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1341919 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1341920 INFO  (jetty-launcher-2850-thread-1) [n:127.0.0.1:56987_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/.
   [junit4]   2> 1342035 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1342040 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1342040 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1342041 INFO  (jetty-launcher-2850-thread-2) [n:127.0.0.1:52489_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/.
   [junit4]   2> 1342066 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1342067 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:49361/solr ready
   [junit4]   2> 1342072 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.TestPullReplica Using legacyCloud?: false
   [junit4]   2> 1342075 INFO  (qtp2071264775-13124) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1342079 INFO  (qtp2071264775-13124) [n:127.0.0.1:56987_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=4
   [junit4]   2> 1342186 INFO  (TEST-TestPullReplica.testCreateDelete-seed#[F0CB252CA943B7CC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete {seed=[F0CB252CA943B7CC:EBDBD4605FB34FFC]}
   [junit4]   2> 1342196 INFO  (OverseerThreadFactory-3744-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_create_delete
   [junit4]   2> 1342198 WARN  (OverseerThreadFactory-3744-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 1342317 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52489/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1342324 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56987/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342331 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52489/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342337 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56987/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342342 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52489/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1342347 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56987/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342352 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52489/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342358 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56987/solr",
   [junit4]   2>   "type":"PULL"}
   [junit4]   2> 1342559 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1342560 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1342560 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342560 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p10&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342561 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1342561 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p12&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342561 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p14&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342561 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n8&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1342561 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342562 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1342685 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1342685 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1342685 INFO  (zkCallback-2861-thread-2-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1342685 INFO  (zkCallback-2862-thread-2-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1343592 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343592 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343597 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343599 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343602 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343606 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p4] Schema name=minimal
   [junit4]   2> 1343609 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343609 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p4' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343609 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.core.pull_replica_test_create_delete.shard1.replica_p4' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343609 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343609 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_p4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_p4/data/]
   [junit4]   2> 1343610 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_n8] Schema name=minimal
   [junit4]   2> 1343614 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343615 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_n8' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343615 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.core.pull_replica_test_create_delete.shard2.replica_n8' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343615 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343615 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_n8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_n8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_n8/data/]
   [junit4]   2> 1343627 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p12] Schema name=minimal
   [junit4]   2> 1343629 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343630 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343649 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p10] Schema name=minimal
   [junit4]   2> 1343649 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p6] Schema name=minimal
   [junit4]   2> 1343650 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343651 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343651 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p10' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343651 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p6' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343651 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p2] Schema name=minimal
   [junit4]   2> 1343651 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.core.pull_replica_test_create_delete.shard2.replica_p10' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343651 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343651 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p10] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p10], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p10/data/]
   [junit4]   2> 1343653 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343653 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p2' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343653 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p14] Schema name=minimal
   [junit4]   2> 1343653 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.core.pull_replica_test_create_delete.shard1.replica_p2' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343653 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343653 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p2/data/]
   [junit4]   2> 1343654 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343654 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p12' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343654 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.core.pull_replica_test_create_delete.shard2.replica_p12' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343654 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343654 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p12] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_p12], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_p12/data/]
   [junit4]   2> 1343654 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343654 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p14' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343655 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.core.pull_replica_test_create_delete.shard2.replica_p14' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343655 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343655 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p14] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p14], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p14/data/]
   [junit4]   2> 1343651 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56987.solr.core.pull_replica_test_create_delete.shard1.replica_p6' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343657 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343657 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p6/data/]
   [junit4]   2> 1343704 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 1343712 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1343714 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1343715 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_n1' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 1343715 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52489.solr.core.pull_replica_test_create_delete.shard1.replica_n1' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@29d749fa
   [junit4]   2> 1343715 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1343715 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_n1/data/]
   [junit4]   2> 1343738 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343738 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343739 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.SolrIndexSearcher Opening [Searcher@2355071b[pull_replica_test_create_delete_shard2_replica_p14] main]
   [junit4]   2> 1343748 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343748 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343749 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@2272aca6[pull_replica_test_create_delete_shard1_replica_p4] main]
   [junit4]   2> 1343752 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343753 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343753 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343753 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343754 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343755 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p4 starting background replication from leader
   [junit4]   2> 1343755 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343756 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343757 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p14 starting background replication from leader
   [junit4]   2> 1343757 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343759 INFO  (searcherExecutor-3749-thread-1-processing-n:127.0.0.1:52489_solr x:pull_replica_test_create_delete_shard1_replica_p4 s:shard1 c:pull_replica_test_create_delete r:core_node7) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p4] Registered new searcher Searcher@2272aca6[pull_replica_test_create_delete_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343760 INFO  (searcherExecutor-3755-thread-1-processing-n:127.0.0.1:56987_solr x:pull_replica_test_create_delete_shard2_replica_p14 s:shard2 c:pull_replica_test_create_delete r:core_node16) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p14] Registered new searcher Searcher@2355071b[pull_replica_test_create_delete_shard2_replica_p14] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343761 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343764 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343788 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343788 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343789 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.SolrIndexSearcher Opening [Searcher@401edf01[pull_replica_test_create_delete_shard1_replica_p2] main]
   [junit4]   2> 1343790 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343791 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343791 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343792 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p2 starting background replication from leader
   [junit4]   2> 1343792 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343792 INFO  (searcherExecutor-3753-thread-1-processing-n:127.0.0.1:56987_solr x:pull_replica_test_create_delete_shard1_replica_p2 s:shard1 c:pull_replica_test_create_delete r:core_node5) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p2] Registered new searcher Searcher@401edf01[pull_replica_test_create_delete_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343793 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343793 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1343793 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1343794 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343794 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343796 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@512d06da[pull_replica_test_create_delete_shard2_replica_n8] main]
   [junit4]   2> 1343805 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343805 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343807 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.SolrIndexSearcher Opening [Searcher@61e1437a[pull_replica_test_create_delete_shard2_replica_p12] main]
   [junit4]   2> 1343811 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343812 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343813 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343813 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578118043026849792
   [junit4]   2> 1343818 INFO  (searcherExecutor-3750-thread-1-processing-n:127.0.0.1:52489_solr x:pull_replica_test_create_delete_shard2_replica_n8 s:shard2 c:pull_replica_test_create_delete r:core_node11) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_n8] Registered new searcher Searcher@512d06da[pull_replica_test_create_delete_shard2_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343826 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1343826 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1343827 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343827 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343829 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1cf7ae8[pull_replica_test_create_delete_shard1_replica_n1] main]
   [junit4]   2> 1343830 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1343830 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1343831 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard2_replica_n8/
   [junit4]   2> 1343831 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1343831 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard2_replica_n8/ has no replicas
   [junit4]   2> 1343831 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1343834 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard2_replica_n8/ shard2
   [junit4]   2> 1343835 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343836 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343836 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343837 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343838 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343840 INFO  (searcherExecutor-3756-thread-1-processing-n:127.0.0.1:52489_solr x:pull_replica_test_create_delete_shard1_replica_n1 s:shard1 c:pull_replica_test_create_delete r:core_node3) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_n1] Registered new searcher Searcher@1cf7ae8[pull_replica_test_create_delete_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343840 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343841 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p12 starting background replication from leader
   [junit4]   2> 1343841 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343841 INFO  (searcherExecutor-3754-thread-1-processing-n:127.0.0.1:52489_solr x:pull_replica_test_create_delete_shard2_replica_p12 s:shard2 c:pull_replica_test_create_delete r:core_node15) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p12] Registered new searcher Searcher@61e1437a[pull_replica_test_create_delete_shard2_replica_p12] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343841 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343842 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578118043057258496
   [junit4]   2> 1343842 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343842 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343843 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.SolrIndexSearcher Opening [Searcher@1538ee1f[pull_replica_test_create_delete_shard2_replica_p10] main]
   [junit4]   2> 1343845 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343845 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343845 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343846 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p10 starting background replication from leader
   [junit4]   2> 1343846 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343846 INFO  (searcherExecutor-3751-thread-1-processing-n:127.0.0.1:56987_solr x:pull_replica_test_create_delete_shard2_replica_p10 s:shard2 c:pull_replica_test_create_delete r:core_node13) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p10] Registered new searcher Searcher@1538ee1f[pull_replica_test_create_delete_shard2_replica_p10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343846 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343848 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1343848 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1343848 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1343848 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1343848 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard1_replica_n1/
   [junit4]   2> 1343849 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1343849 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard1_replica_n1/ has no replicas
   [junit4]   2> 1343849 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1343850 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@1830dc86[pull_replica_test_create_delete_shard1_replica_p6] main]
   [junit4]   2> 1343851 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1343852 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1343853 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1343853 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard1_replica_n1/ shard1
   [junit4]   2> 1343854 INFO  (searcherExecutor-3752-thread-1-processing-n:127.0.0.1:56987_solr x:pull_replica_test_create_delete_shard1_replica_p6 s:shard1 c:pull_replica_test_create_delete r:core_node9) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p6] Registered new searcher Searcher@1830dc86[pull_replica_test_create_delete_shard1_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1343854 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p6 starting background replication from leader
   [junit4]   2> 1343854 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 1343855 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 1343957 INFO  (zkCallback-2861-thread-2-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1343957 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1343957 INFO  (zkCallback-2862-thread-2-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1343957 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1343970 INFO  (indexFetcher-3766-thread-1) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication
   [junit4]   2> 1343985 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1343987 INFO  (qtp1184026906-13136) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n8&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1426
   [junit4]   2> 1343994 INFO  (indexFetcher-3774-thread-1) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication
   [junit4]   2> 1343997 INFO  (qtp1184026906-13134) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p12&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1436
   [junit4]   2> 1343997 INFO  (indexFetcher-3790-thread-1) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication
   [junit4]   2> 1344000 INFO  (qtp2071264775-13130) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p10&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1440
   [junit4]   2> 1344004 INFO  (indexFetcher-3767-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication
   [junit4]   2> 1344007 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1344007 INFO  (indexFetcher-3794-thread-1) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication
   [junit4]   2> 1344009 INFO  (qtp1184026906-13133) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1449
   [junit4]   2> 1344009 INFO  (qtp2071264775-13129) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1448
   [junit4]   2> 1344114 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344114 INFO  (zkCallback-2861-thread-2-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344115 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344115 INFO  (zkCallback-2862-thread-2-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344551 INFO  (indexFetcher-3786-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:52489/solr/pull_replica_test_create_delete_shard2_replica_n8/
   [junit4]   2> 1344552 INFO  (qtp1184026906-13135) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.S.Request [pull_replica_test_create_delete_shard2_replica_n8]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1344552 INFO  (indexFetcher-3786-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 1344552 INFO  (indexFetcher-3786-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 1344552 INFO  (indexFetcher-3786-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 1344552 INFO  (indexFetcher-3786-thread-1) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 1344770 INFO  (qtp2071264775-13125) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p14&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2209
   [junit4]   2> 1344770 INFO  (qtp1184026906-13138) [n:127.0.0.1:52489_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2209
   [junit4]   2> 1344799 INFO  (qtp2071264775-13126) [n:127.0.0.1:56987_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2239
   [junit4]   2> 1344807 INFO  (qtp2071264775-13128) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1344908 INFO  (zkCallback-2861-thread-1-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344908 INFO  (zkCallback-2861-thread-2-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344908 INFO  (zkCallback-2862-thread-2-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1344908 INFO  (zkCallback-2862-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1345810 INFO  (qtp2071264775-13128) [n:127.0.0.1:56987_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/c params={} status=0 QTime=3623
   [junit4]   2> 1345817 INFO  (qtp2071264775-13122) [n:127.0.0.1:56987_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :reload with params name=pull_replica_test_create_delete&action=RELOAD&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1345823 INFO  (OverseerThreadFactory-3744-thread-2-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=RELOAD
   [junit4]   2> 1345825 INFO  (OverseerCollectionConfigSetProcessor-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1345859 INFO  (qtp1184026906-13127)

[...truncated too long message...]

sing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4a8fd2ec: rootName = solr_56987, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@17103388
   [junit4]   2> 1491537 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1491537 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@68bddf4: rootName = solr_52489, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@649f0e5f
   [junit4]   2> 1491538 INFO  (jetty-closer-2851-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1491538 INFO  (jetty-closer-2851-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@ae44352: rootName = solr_56987, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@649f0e5f
   [junit4]   2> 1491542 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1491542 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4285536c: rootName = solr_52489, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@7308d27a
   [junit4]   2> 1491543 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1491545 INFO  (jetty-closer-2851-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1491545 INFO  (jetty-closer-2851-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@29986024: rootName = solr_56987, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@7308d27a
   [junit4]   2> 1491549 INFO  (jetty-closer-2851-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1491552 INFO  (jetty-closer-2851-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98632377539690501-127.0.0.1:52489_solr-n_0000000000) closing
   [junit4]   2> 1491553 INFO  (OverseerStateUpdate-98632377539690501-127.0.0.1:52489_solr-n_0000000000) [n:127.0.0.1:52489_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:52489_solr
   [junit4]   2> 1491556 INFO  (zkCallback-2879-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1491556 INFO  (zkCallback-2879-thread-2-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56987_solr
   [junit4]   2> 1491556 WARN  (zkCallback-2861-thread-5-processing-n:127.0.0.1:52489_solr) [n:127.0.0.1:52489_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1491557 INFO  (jetty-closer-2851-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5e0e0f96{/solr,null,UNAVAILABLE}
   [junit4]   2> 1493058 WARN  (zkCallback-2879-thread-1-processing-n:127.0.0.1:56987_solr) [n:127.0.0.1:56987_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1493059 INFO  (jetty-closer-2851-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1c4a8022{/solr,null,UNAVAILABLE}
   [junit4]   2> 1493062 ERROR (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1493063 INFO  (SUITE-TestPullReplica-seed#[F0CB252CA943B7CC]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49361 49361
   [junit4]   2> 1493149 INFO  (Thread-3169) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49361 49361
   [junit4]   2> 1493156 WARN  (Thread-3169) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 19 /solr/configs/conf
   [junit4]   2> 5 /solr/aliases.json
   [junit4]   2> 4 /solr/security.json
   [junit4]   2> 3 /solr/clusterprops.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 26 /solr/collections/pull_replica_test_create_delete/state.json
   [junit4]   2> 23 /solr/collections/pull_replica_test_kill_pull_replica/state.json
   [junit4]   2> 21 /solr/collections/pull_replica_test_add_remove_pull_replica/state.json
   [junit4]   2> 20 /solr/collections/pull_replica_test_kill_leader/state.json
   [junit4]   2> 20 /solr/collections/pull_replica_test_remove_all_writer_replicas/state.json
   [junit4]   2> 13 /solr/collections/pull_replica_test_add_docs/state.json
   [junit4]   2> 8 /solr/collections/pull_replica_test_real_time_get/state.json
   [junit4]   2> 5 /solr/clusterstate.json
   [junit4]   2> 5 /solr/clusterprops.json
   [junit4]   2> 3 /solr/overseer_elect/election/98632377539690501-127.0.0.1:52489_solr-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 5 /solr/collections
   [junit4]   2> 3 /solr/live_nodes
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_F0CB252CA943B7CC-001
   [junit4]   2> Sep 10, 2017 2:33:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=20349, maxDocsPerChunk=111, blockSize=50), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=20349, blockSize=50)), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-YE, timezone=Asia/Kolkata
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=300691080,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SignatureUpdateProcessorFactoryTest, SolrIndexConfigTest, PeerSyncReplicationTest, LeaderFailoverAfterPartitionTest, TestSQLHandler, ConnectionReuseTest, TestPostingsSolrHighlighter, WrapperMergePolicyFactoryTest, TestZkChroot, BadComponentTest, ConvertedLegacyTest, TestDocTermOrds, TestSystemIdResolver, TestClusterStateMutator, AnalysisErrorHandlingTest, TestFastWriter, TestSolrFieldCacheBean, MoveReplicaHDFSTest, MoreLikeThisHandlerTest, QueryResultKeyTest, BlockDirectoryTest, TestFastLRUCache, TestReqParamsAPI, FileUtilsTest, ConnectionManagerTest, UtilsToolTest, TestMultiValuedNumericRangeQuery, TestRawTransformer, TestPolicyCloud, TestApiFramework, JsonLoaderTest, SolrGraphiteReporterTest, JSONWriterTest, DirectSolrSpellCheckerTest, TestRandomFlRTGCloud, StatelessScriptUpdateProcessorFactoryTest, TestFieldCacheWithThreads, TestComponentsName, TestTrackingShardHandlerFactory, TestAuthenticationFramework, AssignTest, SOLR749Test, MergeStrategyTest, TestSolrConfigHandler, BitVectorTest, TestCloudRecovery, CoreSorterTest, TestFiltering, OpenExchangeRatesOrgProviderTest, TestBadConfig, CloneFieldUpdateProcessorFactoryTest, TestSchemaResource, TestRealTimeGet, SortSpecParsingTest, HLLUtilTest, TestSolrCloudWithSecureImpersonation, DistributedFacetPivotLongTailTest, TestCloudInspectUtil, DistributedQueryElevationComponentTest, HdfsRecoveryZkTest, DistributedSuggestComponentTest, TestSolrCloudWithDelegationTokens, TermsComponentTest, TestExactSharedStatsCache, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, TestRemoteStreaming, TestJoin, DOMUtilTest, LukeRequestHandlerTest, TestSolr4Spatial, UpdateParamsTest, TestConfig, TestFieldSortValues, TestManagedSchemaThreadSafety, HdfsNNFailoverTest, TestSSLRandomization, SearchHandlerTest, TestManagedSynonymFilterFactory, TestLuceneMatchVersion, TolerantUpdateProcessorTest, InfoHandlerTest, TestObjectReleaseTracker, TestElisionMultitermQuery, CoreAdminRequestStatusTest, DistributedIntervalFacetingTest, CustomCollectionTest, TestSystemCollAutoCreate, AnalyticsMergeStrategyTest, CollectionsAPIDistributedZkTest, PreAnalyzedUpdateProcessorTest, TestLMJelinekMercerSimilarityFactory, TestExportWriter, TestSolrDeletionPolicy1, TestOnReconnectListenerSupport, HdfsTlogReplayBufferedWhileIndexingTest, ClassificationUpdateProcessorIntegrationTest, ZkNodePropsTest, TestDistributedGrouping, TestChildDocTransformer, TestDynamicLoading, TestRequestForwarding, TestDistribDocBasedVersion, TestShardHandlerFactory, TestFieldTypeResource, IndexSchemaRuntimeFieldTest, BasicFunctionalityTest, TestExceedMaxTermLength, UpdateLogTest, TestRestManager, TermVectorComponentTest, TestSchemaVersionResource, TestReloadDeadlock, TestCloudManagedSchema, PolyFieldTest, TestRangeQuery, HdfsDirectoryTest, BaseCdcrDistributedZkTest, LoggingHandlerTest, TestLeaderInitiatedRecoveryThread, ExternalFileFieldSortTest, DistributedQueueTest, MBeansHandlerTest, ZkFailoverTest, TestPullReplica]
   [junit4] Completed [255/731 (1!)] on J1 in 152.46s, 10 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 47850 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 167 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/167/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:     1) Thread[id=7374, name=jetty-launcher-1008-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=7378, name=jetty-launcher-1008-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
   1) Thread[id=7374, name=jetty-launcher-1008-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=7378, name=jetty-launcher-1008-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
        at __randomizedtesting.SeedInfo.seed([4DFD666EED4629A5]:0)


FAILED:  org.apache.solr.update.AutoCommitTest.testMaxDocs

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([4DFD666EED4629A5:F47CB0B1C1AC2D2F]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
        at org.apache.solr.update.AutoCommitTest.testMaxDocs(AutoCommitTest.java:225)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
        xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

        request was:q=id:14&qt=&start=0&rows=20&version=2.2
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
        ... 40 more




Build Log:
[...truncated 11492 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_4DFD666EED4629A5-001/init-core-data-001
   [junit4]   2> 836843 WARN  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 836843 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 836845 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 836845 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 836876 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_4DFD666EED4629A5-001/tempDir-001
   [junit4]   2> 836876 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 836877 INFO  (Thread-2928) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 836877 INFO  (Thread-2928) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 836879 ERROR (Thread-2928) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 836977 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:41325
   [junit4]   2> 836993 INFO  (jetty-launcher-1008-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 836993 INFO  (jetty-launcher-1008-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 836998 INFO  (jetty-launcher-1008-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ad484de{/solr,null,AVAILABLE}
   [junit4]   2> 836998 INFO  (jetty-launcher-1008-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21f1aa58{/solr,null,AVAILABLE}
   [junit4]   2> 837004 INFO  (jetty-launcher-1008-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@28b39ff5{HTTP/1.1,[http/1.1]}{127.0.0.1:35102}
   [junit4]   2> 837005 INFO  (jetty-launcher-1008-thread-1) [    ] o.e.j.s.Server Started @840135ms
   [junit4]   2> 837005 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35102}
   [junit4]   2> 837005 ERROR (jetty-launcher-1008-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 837005 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 837005 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 837005 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T10:11:21.562Z
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4e256f14{HTTP/1.1,[http/1.1]}{127.0.0.1:43864}
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.e.j.s.Server Started @840136ms
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43864}
   [junit4]   2> 837006 ERROR (jetty-launcher-1008-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.1.0
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 837006 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T10:11:21.562Z
   [junit4]   2> 837011 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 837015 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 837023 INFO  (jetty-launcher-1008-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41325/solr
   [junit4]   2> 837029 INFO  (jetty-launcher-1008-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41325/solr
   [junit4]   2> 837219 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 837220 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43864_solr
   [junit4]   2> 837221 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.Overseer Overseer (id=98634188812058630-127.0.0.1:43864_solr-n_0000000000) starting
   [junit4]   2> 837222 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 837224 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35102_solr
   [junit4]   2> 837227 INFO  (zkCallback-1019-thread-1-processing-n:127.0.0.1:35102_solr) [n:127.0.0.1:35102_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 837228 INFO  (zkCallback-1020-thread-1-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 837234 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43864_solr
   [junit4]   2> 837236 INFO  (zkCallback-1020-thread-1-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 837236 INFO  (zkCallback-1019-thread-1-processing-n:127.0.0.1:35102_solr) [n:127.0.0.1:35102_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 837274 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 837280 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 837349 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 837374 INFO  (jetty-launcher-1008-thread-1-EventThread) [n:127.0.0.1:35102_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 837401 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 837401 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 837403 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 837406 INFO  (jetty-launcher-1008-thread-2-EventThread) [n:127.0.0.1:43864_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 837507 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 837507 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 837568 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35102.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837571 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43864.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837577 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35102.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837577 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35102.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837578 INFO  (jetty-launcher-1008-thread-1) [n:127.0.0.1:35102_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_4DFD666EED4629A5-001/tempDir-001/node1/.
   [junit4]   2> 837582 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43864.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837582 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43864.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cd3084f
   [junit4]   2> 837583 INFO  (jetty-launcher-1008-thread-2) [n:127.0.0.1:43864_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_4DFD666EED4629A5-001/tempDir-001/node2/.
   [junit4]   2> 837623 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 837624 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41325/solr ready
   [junit4]   2> 837651 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 837696 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=simple-dt&e=1505074282222&s=sv/GMJO4vrJaYytG0rIAb7fevAg="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:22 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:22 GMT
   [junit4]   2> 837697 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 837717 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 852792 WARN  (jetty-launcher-1008-thread-1-SendThread(127.0.0.1:41325)) [n:127.0.0.1:35102_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15213ms for sessionid 0x15e6b45aada0005
   [junit4]   2> 852793 WARN  (jetty-launcher-1008-thread-2-SendThread(127.0.0.1:41325)) [n:127.0.0.1:43864_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15275ms for sessionid 0x15e6b45aada0008
   [junit4]   2> 852793 WARN  (jetty-launcher-1008-thread-1-SendThread(127.0.0.1:41325)) [n:127.0.0.1:35102_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15274ms for sessionid 0x15e6b45aada0007
   [junit4]   2> 852793 WARN  (jetty-launcher-1008-thread-2-SendThread(127.0.0.1:41325)) [n:127.0.0.1:43864_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15209ms for sessionid 0x15e6b45aada0006
   [junit4]   2> 852796 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6b45aada0007, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852805 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6b45aada0008, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852806 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6b45aada0005, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852806 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6b45aada0006, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852807 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=simple-dt&e=1505074282278&s=a2801wCeg9SLOfKDVxiDjbfZdbE="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:22 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:22 GMT
   [junit4]   2> 852807 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 852862 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 852875 INFO  (qtp553025636-7336) [n:127.0.0.1:35102_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 852876 INFO  (qtp553025636-7336) [n:127.0.0.1:35102_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 852877 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=simple-dt&e=1505074297424&s=Du29JsNC668Gh0Ow5gAyUm6DC8Y="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 852878 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 852897 INFO  (jetty-launcher-1008-thread-1-EventThread) [n:127.0.0.1:35102_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 852904 WARN  (zkCallback-1026-thread-2) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@22de099d name: ZooKeeperConnection Watcher:127.0.0.1:41325/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 852904 WARN  (zkCallback-1026-thread-2) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 852906 WARN  (zkCallback-1019-thread-6-processing-n:127.0.0.1:35102_solr) [n:127.0.0.1:35102_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@43a797c5 name: ZooKeeperConnection Watcher:127.0.0.1:41325/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 852906 WARN  (zkCallback-1019-thread-6-processing-n:127.0.0.1:35102_solr) [n:127.0.0.1:35102_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 852906 WARN  (zkCallback-1020-thread-1-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@361d08b7 name: ZooKeeperConnection Watcher:127.0.0.1:41325/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 852907 WARN  (zkCallback-1020-thread-1-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 852965 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 852993 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=simple-dt&e=1505074297524&s=M1gbtn/hoSpDUHk5OoUSD4XRPuw="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 852993 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 853062 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 853075 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=simple-dt&e=1505074297625&s=LFUTZO5qF4zl8xEMTazRc1MuXAE="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 853075 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 853115 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 853189 WARN  (qtp553025636-7332) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2>
   [junit4]   2> 853195 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=simple-dt&e=1505074297683&s=42NUdAZsq7eXBGogtEPP9OCZwtM="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 853195 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 853228 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 853239 INFO  (qtp553025636-7329) [n:127.0.0.1:35102_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 853239 INFO  (qtp553025636-7329) [n:127.0.0.1:35102_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 853240 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=simple-dt&e=1505074297795&s=WsJsdEHeiLnJzAJwtj4BBYiLzbI="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 853240 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 853275 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 853281 INFO  (qtp553025636-7335) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 853281 WARN  (qtp553025636-7335) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication,
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> ... 26 more
   [junit4]   2> 853298 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4] IGNOR/A 0.00s J0 | TestImpersonationWithHadoopAuth.testForwarding
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/HADOOP-9893))
   [junit4]   2> 853340 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 853343 INFO  (qtp553025636-7329) [n:127.0.0.1:35102_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 853343 INFO  (qtp553025636-7329) [n:127.0.0.1:35102_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 853348 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[4DFD666EED4629A5]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=simple-dt&e=1505074297898&s=HCbG8jHDTHZKX+mDzG3BtBRn5IE="; Domain=127.0.0.1; Expires=   , 10-  -2017 20:11:37 GMT; HttpOnly". Invalid 'expires' attribute: , 10- -2017 20:11:37 GMT
   [junit4]   2> 853349 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 854101 WARN  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker-SendThread(127.0.0.1:41325)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15e6b45aada000a has expired
   [junit4]   2> 854101 WARN  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@22de099d name: ZooKeeperConnection Watcher:127.0.0.1:41325/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 854101 WARN  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 854102 WARN  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 854110 INFO  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 854206 ERROR (zkCallback-1026-thread-6) [    ] o.a.s.c.c.ZkStateReader A ZK error has occurred
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:384)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:241)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 854206 ERROR (zkCallback-1026-thread-6) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: A ZK error has occurred
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:244)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:384)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:241)
   [junit4]   2> ... 10 more
   [junit4]   2>
   [junit4]   2> 854206 WARN  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 854206 INFO  (zkCallback-1026-thread-6) [    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 854210 INFO  (jetty-closer-1009-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@28b39ff5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 854210 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1298094081
   [junit4]   2> 854211 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 854211 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@784e3c04: rootName = solr_35102, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@522e9fde
   [junit4]   2> 854211 INFO  (jetty-closer-1009-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@4e256f14{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 854217 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=152669132
   [junit4]   2> 854217 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 854217 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@23a5bda: rootName = solr_43864, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6cdbbe08
   [junit4]   2> 854260 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 854260 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@214e2123: rootName = solr_43864, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5118d82b
   [junit4]   2> 854261 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 854261 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48412c1d: rootName = solr_35102, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5118d82b
   [junit4]   2> 854293 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 854293 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@328ef922: rootName = solr_35102, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6e19c6a9
   [junit4]   2> 854295 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 854295 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3819fc70: rootName = solr_43864, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6e19c6a9
   [junit4]   2> 854296 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 854296 INFO  (jetty-closer-1009-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 854306 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 854310 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 854311 INFO  (jetty-closer-1009-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98634188812058630-127.0.0.1:43864_solr-n_0000000000) closing
   [junit4]   2> 854311 INFO  (OverseerCollectionConfigSetProcessor-98634188812058630-127.0.0.1:43864_solr-n_0000000000) [n:127.0.0.1:43864_solr    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98634188812058630-127.0.0.1:43864_solr-n_0000000000) am no longer a leader.
   [junit4]   2> 854312 INFO  (OverseerStateUpdate-98634188812058630-127.0.0.1:43864_solr-n_0000000000) [n:127.0.0.1:43864_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43864_solr
   [junit4]   2> 854317 INFO  (zkCallback-1020-thread-7-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 854318 ERROR (Thread[Thread-2947,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:35102_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 854360 INFO  (jetty-closer-1009-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@ad484de{/solr,null,UNAVAILABLE}
   [junit4]   2> 854630 INFO  (jetty-launcher-1008-thread-2-EventThread) [n:127.0.0.1:43864_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 855832 WARN  (zkCallback-1020-thread-7-processing-n:127.0.0.1:43864_solr) [n:127.0.0.1:43864_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 855834 ERROR (Thread[Thread-2945,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:43864_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 855849 INFO  (jetty-closer-1009-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21f1aa58{/solr,null,UNAVAILABLE}
   [junit4]   2> 855859 ERROR (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 855860 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41325 41325
   [junit4]   2> 860927 INFO  (Thread-2928) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41325 41325
   [junit4]   2> 860927 WARN  (Thread-2928) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 3 /solr/aliases.json
   [junit4]   2> 3 /solr/clusterprops.json
   [junit4]   2> 2 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 5 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 3 /solr/clusterstate.json
   [junit4]   2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 3 /solr/live_nodes
   [junit4]   2> 3 /solr/collections
   [junit4]   2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2>
   [junit4]   2> 860928 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Sep 10, 2017 10:11:45 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Sep 10, 2017 10:12:05 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
   [junit4]   2>    1) Thread[id=7374, name=jetty-launcher-1008-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2>    2) Thread[id=7378, name=jetty-launcher-1008-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Sep 10, 2017 10:12:05 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=7374, name=jetty-launcher-1008-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>    2) Thread[id=7378, name=jetty-launcher-1008-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2> 881013 ERROR (jetty-launcher-1008-thread-1-EventThread) [n:127.0.0.1:35102_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 881013 ERROR (jetty-launcher-1008-thread-2-EventThread) [n:127.0.0.1:43864_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Sep 10, 2017 10:12:05 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1462, maxMBSortInHeap=6.087067939302372, sim=RandomSimilarity(queryNorm=true): {}, locale=zh-TW, timezone=SystemV/MST7
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=157695272,total=392167424
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrXml, LeaderFailureAfterFreshStartTest, PrimitiveFieldTypeTest, ActionThrottleTest, ChaosMonkeyNothingIsSafeTest, DistributedSpellCheckComponentTest, SoftAutoCommitTest, TestCloudSearcherWarming, CSVRequestHandlerTest, TestMiniSolrCloudClusterSSL, ResourceLoaderTest, DirectUpdateHandlerOptimizeTest, TestConfigReload, SolrInfoBeanTest, CoreAdminCreateDiscoverTest, NumericFieldsTest, TestUtils, MultiTermTest, HdfsLockFactoryTest, ConfigureRecoveryStrategyTest, TestLMDirichletSimilarityFactory, TestRandomDVFaceting, CheckHdfsIndexTest, NoCacheHeaderTest, TestSolrConfigHandlerConcurrent, DisMaxRequestHandlerTest, CreateCollectionCleanupTest, TestQueryWrapperFilter, CloudMLTQParserTest, AnalysisAfterCoreReloadTest, TestRTimerTree, HdfsChaosMonkeyNothingIsSafeTest, TestMacros, SolrCoreMetricManagerTest, TestStressVersions, StatsReloadRaceTest, IndexBasedSpellCheckerTest, TestCloudJSONFacetJoinDomain, HdfsRestartWhileUpdatingTest, SparseHLLTest, TestBlendedInfixSuggestions, SpatialHeatmapFacetsTest, TestScoreJoinQPScore, TestDeleteCollectionOnDownNodes, TestQuerySenderNoQuery, SortByFunctionTest, TestRandomRequestDistribution, TestNumericTokenStream, ZkStateReaderTest, DeleteInactiveReplicaTest, HdfsRecoverLeaseTest, HighlighterConfigTest, HLLSerializationTest, TestDFRSimilarityFactory, CollectionTooManyReplicasTest, AddBlockUpdateTest, TestCollectionAPI, RankQueryTest, TestImpersonationWithHadoopAuth]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=4DFD666EED4629A5 -Dtests.slow=true -Dtests.locale=zh-TW -Dtests.timezone=SystemV/MST7 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestImpersonationWithHadoopAuth (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
   [junit4]    >    1) Thread[id=7374, name=jetty-launcher-1008-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=7378, name=jetty-launcher-1008-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([4DFD666EED4629A5]:0)
   [junit4] Completed [121/731 (1!)] on J0 in 44.19s, 10 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 910 lines...]
   [junit4] Suite: org.apache.solr.update.AutoCommitTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_4DFD666EED4629A5-001/init-core-data-001
   [junit4]   2> 2032302 WARN  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=148 numCloses=148
   [junit4]   2> 2032302 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2032304 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2032304 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2032304 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 2032304 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2032328 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2032343 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 2032539 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2032641 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@31038959
   [junit4]   2> 2032656 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@31038959
   [junit4]   2> 2032657 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@31038959
   [junit4]   2> 2032660 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2032704 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2032735 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2032859 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2032869 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 2032870 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@31038959
   [junit4]   2> 2032870 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2032870 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_4DFD666EED4629A5-001/init-core-data-001/]
   [junit4]   2> 2032878 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=59.6962890625, floorSegmentMB=0.7255859375, forceMergeDeletesPctAllowed=27.277525092100547, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2032947 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2032948 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2032948 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2032948 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2032949 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=67.4443359375, floorSegmentMB=2.1533203125, forceMergeDeletesPctAllowed=29.347690537284322, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2032949 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@296069ba[collection1] main]
   [junit4]   2> 2032950 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2032953 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2032972 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2032972 INFO  (coreLoadExecutor-7596-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578148275484098560
   [junit4]   2> 2032972 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2032978 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2032978 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2032978 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2032978 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2032979 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 2032979 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2032995 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2032995 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
   [junit4]   2> 2032996 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2032996 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2032996 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 2032999 INFO  (searcherExecutor-7597-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@296069ba[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2032999 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@406ddb8f[collection1] realtime]
   [junit4]   2> 2033000 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 4
   [junit4]   2> 2033000 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/core.properties
   [junit4]   2> 2033000 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2033024 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2033039 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2033134 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2033145 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 2033182 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@31038959
   [junit4]   2> 2033182 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_4DFD666EED4629A5-001/init-core-data-001/]
   [junit4]   2> 2033312 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2033312 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2033316 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2e3acb54[collection1] main]
   [junit4]   2> 2033317 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2033318 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2033324 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2033324 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2033324 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578148275853197312
   [junit4]   2> 2033325 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2033325 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2033325 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2033326 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2033326 INFO  (searcherExecutor-7602-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2033327 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=0.2900390

[...truncated too long message...]

31
   [junit4]   2> 2045570 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 2045571 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[4DFD666EED4629A5]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
   [junit4]   2> 2045571 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2045571 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1563523512
   [junit4]   2> 2045571 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2045571 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@391ca337: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@210735e8
   [junit4]   2> 2045575 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2045575 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2975bf48: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@ac6df7a
   [junit4]   2> 2045578 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2045578 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@535617e2: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@45659412
   [junit4]   2> 2045578 INFO  (SUITE-AutoCommitTest-seed#[4DFD666EED4629A5]-worker) [    x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2045578 INFO  (coreCloseExecutor-7614-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@634f0fd4
   [junit4]   2> 2045578 INFO  (coreCloseExecutor-7614-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1666125780
   [junit4]   2> 2045578 INFO  (coreCloseExecutor-7614-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3b36163b: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@678e5b0
   [junit4]   2> 2045588 INFO  (coreCloseExecutor-7614-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 2045588 INFO  (coreCloseExecutor-7614-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7f2e1aab commitCommandVersion:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_4DFD666EED4629A5-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=Lucene50(blocksize=128), field_t=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=MockRandom), subject=PostingsFormat(name=MockRandom), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id_i1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Direct), id_i1=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Lucene70), range_facet_l=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=220, maxMBSortInHeap=5.559237526727993, sim=RandomSimilarity(queryNorm=true): {}, locale=th-TH, timezone=Europe/Zaporozhye
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=182430912,total=507510784
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUCache, CopyFieldTest, TestShortCircuitedRequests, MoreLikeThisHandlerTest, TestCoreDiscovery, BasicAuthStandaloneTest, SynonymTokenizerTest, AssignTest, ConcurrentDeleteAndCreateCollectionTest, DeleteLastCustomShardedReplicaTest, SegmentsInfoRequestHandlerTest, BitVectorTest, SolrPluginUtilsTest, HdfsNNFailoverTest, BasicDistributedZk2Test, TestPolicyCloud, TermsComponentTest, TestCollationFieldDocValues, MoveReplicaHDFSTest, MergeStrategyTest, TestPartialUpdateDeduplication, SpellCheckCollatorTest, TestJsonRequest, TestRawTransformer, TestDefaultStatsCache, HdfsChaosMonkeySafeLeaderTest, TestTolerantUpdateProcessorRandomCloud, TestXmlQParser, DOMUtilTest, TestFieldSortValues, TestHighFrequencyDictionaryFactory, TestDFISimilarityFactory, SolrCmdDistributorTest, TemplateUpdateProcessorTest, TestDelegationWithHadoopAuth, IndexSchemaRuntimeFieldTest, TestRandomFaceting, LoggingHandlerTest, TestManagedSchema, TestTlogReplica, TestInPlaceUpdatesDistrib, ZkNodePropsTest, TestConfigSetProperties, SolrCloudReportersTest, SolrJmxReporterTest, TestMergePolicyConfig, TestSystemIdResolver, FileBasedSpellCheckerTest, TestLeaderInitiatedRecoveryThread, TestReRankQParserPlugin, SolrRequestParserTest, TestPayloadScoreQParserPlugin, MoveReplicaTest, TestDocSet, TestExclusionRuleCollectionAccess, TestDistributedStatsComponentCardinality, DocValuesNotIndexedTest, TestInitQParser, TestCollectionsAPIViaSolrCloudCluster, SolrGraphiteReporterTest, TestLMJelinekMercerSimilarityFactory, DistributedFacetPivotSmallTest, SystemInfoHandlerTest, AnalysisErrorHandlingTest, DeleteStatusTest, TestStressUserVersions, MetricsConfigTest, ClassificationUpdateProcessorIntegrationTest, TestSolrJ, TestBadConfig, TestUseDocValuesAsStored, TestLeaderElectionWithEmptyReplica, SchemaVersionSpecificBehaviorTest, HttpSolrCallGetCoreTest, InfixSuggestersTest, TestIndexSearcher, HighlighterMaxOffsetTest, TestBlobHandler, MBeansHandlerTest, SolrMetricsIntegrationTest, DateRangeFieldTest, TestQueryTypes, TestManagedResource, ExternalFileFieldSortTest, TestOverriddenPrefixQueryForCustomFieldType, TestGraphTermsQParserPlugin, AddSchemaFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestSolrCloudWithHadoopAuthPlugin, TestSchemaResource, TestApiFramework, DistributedMLTComponentTest, DistributedQueryElevationComponentTest, EchoParamsTest, TestRemoteStreaming, FieldMutatingUpdateProcessorTest, TestLockTree, TestFieldCacheSortRandom, TestSolr4Spatial2, BlockJoinFacetSimpleTest, TestConfig, CollectionStateFormat2Test, TestScoreJoinQPNoScore, HttpPartitionTest, TestSortByMinMaxFunction, TestDynamicFieldResource, DistributedSuggestComponentTest, TestRangeQuery, HdfsWriteToMultipleCollectionsTest, ZkSolrClientTest, LeaderInitiatedRecoveryOnCommitTest, BadCopyFieldTest, HdfsSyncSliceTest, PreAnalyzedFieldTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestNRTOpen, DistanceUnitsTest, TestFieldTypeResource, TermVectorComponentTest, SolrCoreCheckLockOnStartupTest, TestElisionMultitermQuery, ExitableDirectoryReaderTest, TestSSLRandomization, TestSubQueryTransformerCrossCore, TestBinaryResponseWriter, TestSchemaManager, TestLegacyField, ConnectionReuseTest, QueryEqualityTest, JSONWriterTest, TestSolrCloudSnapshots, TestLegacyNumericUtils, SolrIndexConfigTest, CdcrReplicationDistributedZkTest, TestRebalanceLeaders, TestSolrFieldCacheBean, TestShardHandlerFactory, SolrMetricManagerTest, TestInfoStreamLogging, SearchHandlerTest, SharedFSAutoReplicaFailoverTest, BlobRepositoryCloudTest, DocumentAnalysisRequestHandlerTest, TestPushWriter, ShardRoutingCustomTest, TestTrie, TestRandomCollapseQParserPlugin, TestConfigOverlay, TestFastOutputStream, TestRuleBasedAuthorizationPlugin, HLLUtilTest, CloneFieldUpdateProcessorFactoryTest, TestSystemCollAutoCreate, V2StandaloneTest, SubstringBytesRefFilterTest, TestPostingsSolrHighlighter, TestExportWriter, TestOnReconnectListenerSupport, TestMultiValuedNumericRangeQuery, HdfsTlogReplayBufferedWhileIndexingTest, TermVectorComponentDistributedTest, TestManagedSchemaThreadSafety, TestExceedMaxTermLength, TestSolrDeletionPolicy1, SolrIndexMetricsTest, TestRecovery, TestDocTermOrds, BlockDirectoryTest, TestBM25SimilarityFactory, SpellingQueryConverterTest, DebugComponentTest, TestLuceneMatchVersion, SimpleCollectionCreateDeleteTest, TestReloadDeadlock, ShowFileRequestHandlerTest, DirectSolrSpellCheckerTest, CdcrRequestHandlerTest, LeaderElectionIntegrationTest, HdfsDirectoryTest, TestPerFieldSimilarityWithDefaultOverride, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, SyncSliceTest, OverseerTest, TestReplicationHandler, AutoCommitTest]
   [junit4] Completed [380/731 (2!)] on J1 in 13.32s, 3 tests, 1 error <<< FAILURES!

[...truncated 47449 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 168 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/168/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream

Error Message:
Error from server at http://127.0.0.1:36040/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:36040/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([E6101A2441929E4A:C4D09BDF62F8B45A]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream(StreamExpressionTest.java:7142)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36040/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more




Build Log:
[...truncated 14490 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.io.stream.StreamExpressionTest
   [junit4]   2> 151164 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/init-core-data-001
   [junit4]   2> 151164 WARN  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 151164 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 151165 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 151166 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001
   [junit4]   2> 151166 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 151166 INFO  (Thread-657) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 151166 INFO  (Thread-657) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 151167 ERROR (Thread-657) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 151281 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:62410
   [junit4]   2> 151290 INFO  (jetty-launcher-987-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151290 INFO  (jetty-launcher-987-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151292 INFO  (jetty-launcher-987-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151292 INFO  (jetty-launcher-987-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151293 INFO  (jetty-launcher-987-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4c48e098{/solr,null,AVAILABLE}
   [junit4]   2> 151295 INFO  (jetty-launcher-987-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@30e05417{/solr,null,AVAILABLE}
   [junit4]   2> 151295 INFO  (jetty-launcher-987-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@171d94ab{/solr,null,AVAILABLE}
   [junit4]   2> 151300 INFO  (jetty-launcher-987-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2fe45924{/solr,null,AVAILABLE}
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2f5653d6{HTTP/1.1,[http/1.1]}{127.0.0.1:๓๖๐๔๐}
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.e.j.s.Server Started @๑๕๓๖๘๒ms
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36040}
   [junit4]   2> 151308 ERROR (jetty-launcher-987-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4cba9168{HTTP/1.1,[http/1.1]}{127.0.0.1:๕๕๔๗๐}
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T21:32:29.639Z
   [junit4]   2> 151308 INFO  (jetty-launcher-987-thread-1) [    ] o.e.j.s.Server Started @๑๕๓๖๘๓ms
   [junit4]   2> 151309 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55470}
   [junit4]   2> 151309 ERROR (jetty-launcher-987-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151309 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 151309 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151309 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151309 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T21:32:29.640Z
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@227401ac{HTTP/1.1,[http/1.1]}{127.0.0.1:๖๒๓๑๕}
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.e.j.s.Server Started @๑๕๓๖๘๖ms
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62315}
   [junit4]   2> 151312 ERROR (jetty-launcher-987-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151312 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T21:32:29.643Z
   [junit4]   2> 151314 INFO  (jetty-launcher-987-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5d7e45a9{HTTP/1.1,[http/1.1]}{127.0.0.1:๕๕๔๑๑}
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.e.j.s.Server Started @๑๕๓๖๘๙ms
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55411}
   [junit4]   2> 151315 ERROR (jetty-launcher-987-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151315 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-10T21:32:29.646Z
   [junit4]   2> 151319 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151321 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151321 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151336 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6db543ba0003, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 151337 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6db543ba0001, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 151337 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6db543ba0002, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 151337 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151341 INFO  (jetty-launcher-987-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62410/solr
   [junit4]   2> 151352 INFO  (jetty-launcher-987-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62410/solr
   [junit4]   2> 151353 INFO  (jetty-launcher-987-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62410/solr
   [junit4]   2> 151357 INFO  (jetty-launcher-987-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62410/solr
   [junit4]   2> 151410 WARN  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151410 WARN  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151412 WARN  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151412 WARN  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151423 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151423 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151423 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151423 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151424 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62315_solr
   [junit4]   2> 151428 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55470_solr
   [junit4]   2> 151429 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55411_solr
   [junit4]   2> 151429 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.Overseer Overseer (id=98636867141500939-127.0.0.1:62315_solr-n_0000000000) starting
   [junit4]   2> 151455 INFO  (zkCallback-1009-thread-2-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151461 INFO  (zkCallback-997-thread-1-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 151462 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 151462 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151463 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151464 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36040_solr
   [junit4]   2> 151465 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 151474 INFO  (zkCallback-1009-thread-1-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 151482 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 151482 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 151488 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62315_solr
   [junit4]   2> 151492 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 151492 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 151493 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 151493 INFO  (zkCallback-1009-thread-1-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 151576 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55411.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151584 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55411.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151584 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55411.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151585 INFO  (jetty-launcher-987-thread-3) [n:127.0.0.1:55411_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node4/.
   [junit4]   2> 151607 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55470.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151618 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55470.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151618 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55470.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151619 INFO  (jetty-launcher-987-thread-1) [n:127.0.0.1:55470_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node1/.
   [junit4]   2> 151643 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36040.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151648 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36040.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151648 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36040.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151654 INFO  (jetty-launcher-987-thread-4) [n:127.0.0.1:36040_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node3/.
   [junit4]   2> 151659 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62315.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151668 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62315.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151669 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62315.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 151670 INFO  (jetty-launcher-987-thread-2) [n:127.0.0.1:62315_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node2/.
   [junit4]   2> 151717 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 151718 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:62410/solr ready
   [junit4]   2> 151730 INFO  (qtp2072545859-2931) [n:127.0.0.1:55411_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 151733 INFO  (OverseerThreadFactory-859-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 151845 INFO  (OverseerStateUpdate-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62315/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 151848 INFO  (OverseerStateUpdate-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36040/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 152058 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 152059 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 152062 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 152062 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 152179 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152179 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152180 INFO  (zkCallback-1007-thread-2-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152181 INFO  (zkCallback-1008-thread-2-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153098 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 153098 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 153114 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 153114 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema [collection1_shard2_replica_n2] Schema name=test
   [junit4]   2> 153192 WARN  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 153192 WARN  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 153192 WARN  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 153192 WARN  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 153193 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 153193 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 153193 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62315.solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 153193 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 153193 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node2/collection1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node2/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 153199 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 153199 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n2' using configuration from collection collection1, trusted=true
   [junit4]   2> 153199 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36040.solr.core.collection1.shard2.replica_n2' (registry 'solr.core.collection1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 153200 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 153200 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [[collection1_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node3/collection1_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node3/./collection1_shard2_replica_n2/data/]
   [junit4]   2> 153250 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 153250 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 153251 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 153251 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 153252 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e7fa89[collection1_shard1_replica_n1] main]
   [junit4]   2> 153254 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 153255 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 153255 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 153255 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 153256 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 153256 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578189876454490112
   [junit4]   2> 153265 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 153265 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 153267 INFO  (searcherExecutor-868-thread-1-processing-n:127.0.0.1:62315_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@5e7fa89[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 153267 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@716e516b[collection1_shard2_replica_n2] main]
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62315/solr/collection1_shard1_replica_n1/
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:62315/solr/collection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 153274 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 153278 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62315/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 153280 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 153280 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 153284 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 153284 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578189876483850240
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/ has no replicas
   [junit4]   2> 153293 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 153299 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/ shard2
   [junit4]   2> 153300 INFO  (searcherExecutor-869-thread-1-processing-n:127.0.0.1:36040_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@716e516b[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 153400 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153400 INFO  (zkCallback-1007-thread-2-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153400 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153400 INFO  (zkCallback-1008-thread-2-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153429 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 153434 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1376
   [junit4]   2> 153450 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 153454 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1392
   [junit4]   2> 153459 INFO  (qtp2072545859-2931) [n:127.0.0.1:55411_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 153555 INFO  (zkCallback-1008-thread-2-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153555 INFO  (zkCallback-1007-thread-2-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153555 INFO  (zkCallback-1007-thread-1-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153556 INFO  (zkCallback-1008-thread-1-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153737 INFO  (OverseerCollectionConfigSetProcessor-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 154462 INFO  (qtp2072545859-2931) [n:127.0.0.1:55411_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2733
   [junit4]   2> 154462 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):90
   [junit4]   2> 154463 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 154487 INFO  (TEST-StreamExpressionTest.testFeaturesSelectionStream-seed#[E6101A2441929E4A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testFeaturesSelectionStream
   [junit4]   2> 154498 INFO  (qtp1879061312-2962) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/&wt=javabin&version=2}{deleteByQuery=*:* (-1578189877753675776)} 0 2
   [junit4]   2> 154500 INFO  (qtp1879061312-2960) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189877758918656,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 154500 INFO  (qtp1879061312-2960) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 154500 INFO  (qtp1879061312-2960) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 154500 INFO  (qtp1879061312-2960) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
   [junit4]   2> 154501 INFO  (qtp1149202594-2941) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189877759967232,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 154501 INFO  (qtp1149202594-2941) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 154502 INFO  (qtp1149202594-2941) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 154502 INFO  (qtp1149202594-2941) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:36040/solr/collection1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 154503 INFO  (qtp1149202594-2946) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=collection1:4&commit=true&wt=javabin&version=2}{deleteByQuery=*:* (-1578189877749481472),commit=} 0 12
   [junit4]   2> 154504 INFO  (qtp2072545859-2937) [n:127.0.0.1:55411_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=ml&name=destinationCollection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 154506 INFO  (OverseerThreadFactory-859-thread-2-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.CreateCollectionCmd Create collection destinationCollection
   [junit4]   2> 154718 INFO  (OverseerStateUpdate-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"destinationCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"destinationCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55411/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 154721 INFO  (OverseerStateUpdate-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"destinationCollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"destinationCollection_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:55470/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 154925 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=ml&newCollection=true&name=destinationCollection_shard1_replica_n1&action=CREATE&numShards=2&collection=destinationCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 154926 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 154932 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=ml&newCollection=true&name=destinationCollection_shard2_replica_n2&action=CREATE&numShards=2&collection=destinationCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 154933 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 155048 INFO  (zkCallback-1009-thread-1-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155048 INFO  (zkCallback-1009-thread-2-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155048 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155048 INFO  (zkCallback-997-thread-1-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155945 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 155959 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 155966 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.s.IndexSchema [destinationCollection_shard1_replica_n1] Schema name=test
   [junit4]   2> 155968 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.s.IndexSchema [destinationCollection_shard2_replica_n2] Schema name=test
   [junit4]   2> 155973 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 155973 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'destinationCollection_shard1_replica_n1' using configuration from collection destinationCollection, trusted=true
   [junit4]   2> 155974 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55411.solr.core.destinationCollection.shard1.replica_n1' (registry 'solr.core.destinationCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 155974 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 155974 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SolrCore [[destinationCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node4/destinationCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node4/./destinationCollection_shard1_replica_n1/data/]
   [junit4]   2> 155975 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 155975 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'destinationCollection_shard2_replica_n2' using configuration from collection destinationCollection, trusted=true
   [junit4]   2> 155975 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55470.solr.core.destinationCollection.shard2.replica_n2' (registry 'solr.core.destinationCollection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c4dfd4
   [junit4]   2> 155976 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 155976 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SolrCore [[destinationCollection_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node1/destinationCollection_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001/tempDir-001/node1/./destinationCollection_shard2_replica_n2/data/]
   [junit4]   2> 156092 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 156092 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 156092 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 156092 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 156093 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 156093 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 156093 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 156093 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 156094 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@df26ebb[destinationCollection_shard1_replica_n1] main]
   [junit4]   2> 156094 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@106fba3d[destinationCollection_shard2_replica_n2] main]
   [junit4]   2> 156095 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/ml
   [junit4]   2> 156096 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/ml
   [junit4]   2> 156096 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/ml
   [junit4]   2> 156096 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/ml
   [junit4]   2> 156097 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 156097 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 156097 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578189879433494528
   [junit4]   2> 156097 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578189879433494528
   [junit4]   2> 156097 INFO  (searcherExecutor-878-thread-1-processing-n:127.0.0.1:55411_solr x:destinationCollection_shard1_replica_n1 s:shard1 c:destinationCollection r:core_node3) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SolrCore [destinationCollection_shard1_replica_n1] Registered new searcher Searcher@df26ebb[destinationCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 156099 INFO  (searcherExecutor-879-thread-1-processing-n:127.0.0.1:55470_solr x:destinationCollection_shard2_replica_n2 s:shard2 c:destinationCollection r:core_node4) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SolrCore [destinationCollection_shard2_replica_n2] Registered new searcher Searcher@106fba3d[destinationCollection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55470/solr/destinationCollection_shard2_replica_n2/
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:55470/solr/destinationCollection_shard2_replica_n2/ has no replicas
   [junit4]   2> 156107 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55411/solr/destinationCollection_shard1_replica_n1/
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:55411/solr/destinationCollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 156115 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 156115 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55470/solr/destinationCollection_shard2_replica_n2/ shard2
   [junit4]   2> 156120 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55411/solr/destinationCollection_shard1_replica_n1/ shard1
   [junit4]   2> 156227 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156227 INFO  (zkCallback-1009-thread-2-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156227 INFO  (zkCallback-1009-thread-1-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156227 INFO  (zkCallback-997-thread-1-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156266 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 156269 INFO  (qtp897201867-2952) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=ml&newCollection=true&name=destinationCollection_shard2_replica_n2&action=CREATE&numShards=2&collection=destinationCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1337
   [junit4]   2> 156275 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 156278 INFO  (qtp2072545859-2935) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=ml&newCollection=true&name=destinationCollection_shard1_replica_n1&action=CREATE&numShards=2&collection=destinationCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1353
   [junit4]   2> 156285 INFO  (qtp2072545859-2937) [n:127.0.0.1:55411_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 156386 INFO  (zkCallback-997-thread-1-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156386 INFO  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156386 INFO  (zkCallback-1009-thread-2-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156386 INFO  (zkCallback-1009-thread-1-processing-n:127.0.0.1:55411_solr) [n:127.0.0.1:55411_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destinationCollection/state.json] for collection [destinationCollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156509 INFO  (OverseerCollectionConfigSetProcessor-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 157289 INFO  (qtp2072545859-2937) [n:127.0.0.1:55411_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=ml&name=destinationCollection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2786
   [junit4]   2> 157290 INFO  (TEST-StreamExpressionTest.testFeaturesSelectionStream-seed#[E6101A2441929E4A]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: destinationCollection failOnTimeout:true timeout (sec):90
   [junit4]   2> 157290 INFO  (TEST-StreamExpressionTest.testFeaturesSelectionStream-seed#[E6101A2441929E4A]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: destinationCollection
   [junit4]   2> 157441 INFO  (qtp1149202594-2942) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=collection1:4&wt=javabin&version=2}{add=[2 (1578189880695980032), 3 (1578189880700174336), 5 (1578189880700174337), 6 (1578189880700174338), 7 (1578189880700174339), 9 (1578189880700174340), 17 (1578189880701222912), 18 (1578189880701222913), 19 (1578189880701222914), 21 (1578189880701222915), ... (2410 adds)]} 0 140
   [junit4]   2> 157458 INFO  (qtp1879061312-2964) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=collection1:4&wt=javabin&version=2}{add=[0 (1578189880698077184), 1 (1578189880703320064), 4 (1578189880703320065), 8 (1578189880703320066), 10 (1578189880703320067), 11 (1578189880704368640), 12 (1578189880704368641), 13 (1578189880704368642), 14 (1578189880704368643), 15 (1578189880704368644), ... (2590 adds)]} 0 155
   [junit4]   2> 157462 INFO  (qtp1879061312-2965) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189880864800768,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 157463 INFO  (qtp1879061312-2965) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@474f73a7 commitCommandVersion:1578189880864800768
   [junit4]   2> 157464 INFO  (qtp1149202594-2944) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189880866897920,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 157465 INFO  (qtp1149202594-2944) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7c7b7dd1 commitCommandVersion:1578189880866897920
   [junit4]   2> 157522 INFO  (qtp1879061312-2965) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d2dbf8[collection1_shard1_replica_n1] main]
   [junit4]   2> 157522 INFO  (qtp1879061312-2965) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 157522 INFO  (qtp1149202594-2944) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@51446061[collection1_shard2_replica_n2] main]
   [junit4]   2> 157523 INFO  (qtp1149202594-2944) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 157523 INFO  (searcherExecutor-868-thread-1-processing-n:127.0.0.1:62315_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@3d2dbf8[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):c2590)))}
   [junit4]   2> 157523 INFO  (qtp1879061312-2965) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62315/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 60
   [junit4]   2> 157523 INFO  (searcherExecutor-869-thread-1-processing-n:127.0.0.1:36040_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@51446061[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):c2410)))}
   [junit4]   2> 157524 INFO  (qtp1149202594-2944) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62315/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 59
   [junit4]   2> 157525 INFO  (qtp1879061312-2963) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=collection1:4&commit=true&wt=javabin&version=2}{commit=} 0 65
   [junit4]   2> 157530 INFO  (TEST-StreamExpressionTest.testFeaturesSelectionStream-seed#[E6101A2441929E4A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 157531 INFO  (TEST-StreamExpressionTest.testFeaturesSelectionStream-seed#[E6101A2441929E4A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:62410/solr ready
   [junit4]   2> 157560 INFO  (qtp1149202594-2938) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/select params={q=*:*&distrib=false&positiveLabel=1&field=whitetok&numTerms=4&fq={!igain}&wt=javabin&version=2&outcome=out_i} hits=2410 status=0 QTime=25
   [junit4]   2> 157561 INFO  (qtp1879061312-2959) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&distrib=false&positiveLabel=1&field=whitetok&numTerms=4&fq={!igain}&wt=javabin&version=2&outcome=out_i} hits=2590 status=0 QTime=26
   [junit4]   2> 157570 INFO  (qtp1879061312-2956) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&distrib=false&positiveLabel=1&field=whitetok&numTerms=4&fq={!igain}&wt=javabin&version=2&outcome=out_i} hits=2590 status=0 QTime=3
   [junit4]   2> 157571 INFO  (qtp1149202594-2945) [n:127.0.0.1:36040_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/select params={q=*:*&distrib=false&positiveLabel=1&field=whitetok&numTerms=4&fq={!igain}&wt=javabin&version=2&outcome=out_i} hits=2410 status=0 QTime=3
   [junit4]   2> 157578 INFO  (qtp897201867-2950) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [destinationCollection_shard2_replica_n2]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[first_1 (1578189880983289856), first_2 (1578189880986435584), first_4 (1578189880986435585)]} 0 2
   [junit4]   2> 157581 INFO  (qtp2072545859-2939) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [destinationCollection_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[first_3 (1578189880987484160)]} 0 2
   [junit4]   2> 157585 INFO  (qtp897201867-2954) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189880993775616,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 157585 INFO  (qtp897201867-2954) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@621256f commitCommandVersion:1578189880993775616
   [junit4]   2> 157591 INFO  (qtp897201867-2954) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@26159093[destinationCollection_shard2_replica_n2] main]
   [junit4]   2> 157592 INFO  (qtp897201867-2954) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 157592 INFO  (qtp2072545859-2933) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1578189881001115648,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 157592 INFO  (qtp2072545859-2933) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@77872a2 commitCommandVersion:1578189881001115648
   [junit4]   2> 157594 INFO  (searcherExecutor-879-thread-1-processing-n:127.0.0.1:55470_solr x:destinationCollection_shard2_replica_n2 s:shard2 c:destinationCollection r:core_node4) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.c.SolrCore [destinationCollection_shard2_replica_n2] Registered new searcher Searcher@26159093[destinationCollection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):c3)))}
   [junit4]   2> 157594 INFO  (qtp897201867-2954) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [destinationCollection_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55470/solr/destinationCollection_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 8
   [junit4]   2> 157598 INFO  (qtp2072545859-2933) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@aa8229f[destinationCollection_shard1_replica_n1] main]
   [junit4]   2> 157599 INFO  (qtp2072545859-2933) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 157600 INFO  (searcherExecutor-878-thread-1-processing-n:127.0.0.1:55411_solr x:destinationCollection_shard1_replica_n1 s:shard1 c:destinationCollection r:core_node3) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.c.SolrCore [destinationCollection_shard1_replica_n1] Registered new searcher Searcher@aa8229f[destinationCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):c1)))}
   [junit4]   2> 157600 INFO  (qtp2072545859-2933) [n:127.0.0.1:55411_solr c:destinationCollection s:shard1 r:core_node3 x:destinationCollection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [destinationCollection_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55470/solr/destinationCollection_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 7
   [junit4]   2> 157601 INFO  (qtp897201867-2953) [n:127.0.0.1:55470_solr c:destinationCollection s:shard2 r:core_node4 x:destinationCollection_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorF

[...truncated too long message...]

=763367838
   [junit4]   2> 254174 INFO  (coreCloseExecutor-1059-thread-2) [n:127.0.0.1:36040_solr c:mainCorpus s:shard2 r:core_node4 x:mainCorpus_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b1a055f: rootName = solr_36040, domain = solr.core.mainCorpus.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.mainCorpus.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@7351ae61
   [junit4]   2> 254174 INFO  (zkCallback-1008-thread-4-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 254217 INFO  (coreCloseExecutor-1060-thread-1) [n:127.0.0.1:55470_solr c:destination s:shard1 r:core_node3 x:destination_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.destination.shard1.leader, tag=967950193
   [junit4]   2> 254231 INFO  (zkCallback-1007-thread-3-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 254232 INFO  (zkCallback-1008-thread-4-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 254238 INFO  (coreCloseExecutor-1059-thread-2) [n:127.0.0.1:36040_solr c:mainCorpus s:shard2 r:core_node4 x:mainCorpus_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard2.leader, tag=763367838
   [junit4]   2> 254239 INFO  (coreCloseExecutor-1061-thread-1) [n:127.0.0.1:62315_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=156373617
   [junit4]   2> 254241 INFO  (coreCloseExecutor-1061-thread-3) [n:127.0.0.1:62315_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.mainCorpus.shard1.replica_n1, tag=2086939388
   [junit4]   2> 254241 INFO  (coreCloseExecutor-1061-thread-3) [n:127.0.0.1:62315_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@be3faac: rootName = solr_62315, domain = solr.core.mainCorpus.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.mainCorpus.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@1aa3722b
   [junit4]   2> 254247 INFO  (zkCallback-1008-thread-4-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 254254 INFO  (coreCloseExecutor-1061-thread-3) [n:127.0.0.1:62315_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard1.leader, tag=2086939388
   [junit4]   2> 254254 INFO  (coreCloseExecutor-1061-thread-2) [n:127.0.0.1:62315_solr c:workQueue s:shard1 r:core_node2 x:workQueue_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.workQueue.shard1.replica_n1, tag=1664442876
   [junit4]   2> 254255 INFO  (coreCloseExecutor-1061-thread-2) [n:127.0.0.1:62315_solr c:workQueue s:shard1 r:core_node2 x:workQueue_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@294a3162: rootName = solr_62315, domain = solr.core.workQueue.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.workQueue.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2ca99d52
   [junit4]   2> 254264 INFO  (coreCloseExecutor-1061-thread-2) [n:127.0.0.1:62315_solr c:workQueue s:shard1 r:core_node2 x:workQueue_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.workQueue.shard1.leader, tag=1664442876
   [junit4]   2> 254264 INFO  (coreCloseExecutor-1061-thread-2) [n:127.0.0.1:62315_solr c:workQueue s:shard1 r:core_node2 x:workQueue_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 254264 INFO  (coreCloseExecutor-1061-thread-2) [n:127.0.0.1:62315_solr c:workQueue s:shard1 r:core_node2 x:workQueue_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2651f622 commitCommandVersion:0
   [junit4]   2> 254280 INFO  (jetty-closer-988-thread-4) [    ] o.a.s.c.Overseer Overseer (id=98636867141500939-127.0.0.1:62315_solr-n_0000000000) closing
   [junit4]   2> 254280 INFO  (OverseerStateUpdate-98636867141500939-127.0.0.1:62315_solr-n_0000000000) [n:127.0.0.1:62315_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:62315_solr
   [junit4]   2> 255730 WARN  (zkCallback-997-thread-2-processing-n:127.0.0.1:55470_solr) [n:127.0.0.1:55470_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 255731 INFO  (jetty-closer-988-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30e05417{/solr,null,UNAVAILABLE}
   [junit4]   2> 255747 WARN  (zkCallback-1007-thread-3-processing-n:127.0.0.1:36040_solr) [n:127.0.0.1:36040_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 255748 INFO  (jetty-closer-988-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4c48e098{/solr,null,UNAVAILABLE}
   [junit4]   2> 255787 WARN  (zkCallback-1008-thread-4-processing-n:127.0.0.1:62315_solr) [n:127.0.0.1:62315_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 255788 INFO  (jetty-closer-988-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2fe45924{/solr,null,UNAVAILABLE}
   [junit4]   2> 255791 ERROR (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 255792 INFO  (SUITE-StreamExpressionTest-seed#[E6101A2441929E4A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:62410 62410
   [junit4]   2> 256475 INFO  (Thread-657) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:62410 62410
   [junit4]   2> 256477 WARN  (Thread-657) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 92 /solr/aliases.json
   [junit4]   2> 92 /solr/clusterprops.json
   [junit4]   2> 10 /solr/configs/conf
   [junit4]   2> 6 /solr/configs/ml
   [junit4]   2> 4 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 92 /solr/clusterstate.json
   [junit4]   2> 40 /solr/collections/destinationCollection/state.json
   [junit4]   2> 26 /solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 22 /solr/collections/destination/state.json
   [junit4]   2> 22 /solr/collections/mainCorpus/state.json
   [junit4]   2> 20 /solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 20 /solr/collections/workQueue/state.json
   [junit4]   2> 12 /solr/collections/collection1/state.json
   [junit4]   2> 10 /solr/collections/uknownCollection/state.json
   [junit4]   2> 10 /solr/collections/modelCollection/state.json
   [junit4]   2> 10 /solr/collections/checkpointCollection/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98636867141500939-127.0.0.1:62315_solr-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 92 /solr/live_nodes
   [junit4]   2> 92 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_E6101A2441929E4A-001
   [junit4]   2> Sep 10, 2017 9:34:14 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=Pacific/Guam
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=106828584,total=201232384
   [junit4]   2> NOTE: All tests run in this JVM: [LargeVolumeBinaryJettyTest, MultiplyEvaluatorTest, AppendEvaluatorTest, ContentStreamTest, ConcatOperationTest, TestCloudSolrClientConnections, TestLBHttpSolrClient, ExclusiveOrEvaluatorTest, UuidEvaluatorTest, TestPolicy, LessThanEvaluatorTest, StreamingTest, MergeIndexesEmbeddedTest, NoOpResponseParserTest, CommonAdminParamsTest, DocumentAnalysisResponseTest, FloorEvaluatorTest, QueryResponseTest, CosineEvaluatorTest, GetByIdTest, SolrExceptionTest, LengthEvaluatorTest, TestDelegationTokenRequest, SelectWithEvaluatorsTest, TestJsonRecordReader, SchemaTest, EmpiricalDistributionEvaluatorTest, PowerEvaluatorTest, SolrExampleBinaryTest, ShardParamsTest, GraphTest, RecursiveEvaluatorTest, TestDelegationTokenResponse, TestToleratedUpdateError, ConcurrentUpdateSolrClientBuilderTest, ConcurrentUpdateSolrClientTest, HttpSolrClientBuilderTest, HttpSolrClientSSLAuthConPoolTest, JdbcDriverTest, JdbcTest, StreamExpressionTest]
   [junit4] Completed [145/145 (1!)] on J1 in 105.60s, 108 tests, 1 error <<< FAILURES!

[...truncated 45356 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 169 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/169/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete

Error Message:
Error from server at http://127.0.0.1:48038/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:48038/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([24E1A3E1F5AF5EDC:871B0D447247B479]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete(TestCollectionsAPIViaSolrCloudCluster.java:167)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:48038/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more




Build Log:
[...truncated 12870 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/init-core-data-001
   [junit4]   2> 2984135 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[24E1A3E1F5AF5EDC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2984137 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[24E1A3E1F5AF5EDC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2984137 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[24E1A3E1F5AF5EDC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2984145 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001
   [junit4]   2> 2984145 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2984146 INFO  (Thread-4608) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2984146 INFO  (Thread-4608) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2984150 ERROR (Thread-4608) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2984246 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer start zk server on port:53839
   [junit4]   2> 2984285 INFO  (jetty-launcher-2759-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2984285 INFO  (jetty-launcher-2759-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2984286 INFO  (jetty-launcher-2759-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2984288 INFO  (jetty-launcher-2759-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2984288 INFO  (jetty-launcher-2759-thread-5) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2984289 INFO  (jetty-launcher-2759-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f1474ac{/solr,null,AVAILABLE}
   [junit4]   2> 2984290 INFO  (jetty-launcher-2759-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@10d5a0ca{HTTP/1.1,[http/1.1]}{127.0.0.1:35734}
   [junit4]   2> 2984290 INFO  (jetty-launcher-2759-thread-3) [    ] o.e.j.s.Server Started @2987666ms
   [junit4]   2> 2984290 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35734}
   [junit4]   2> 2984290 ERROR (jetty-launcher-2759-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2984291 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2984291 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2984291 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2984291 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:41.545Z
   [junit4]   2> 2984291 INFO  (jetty-launcher-2759-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@35c16ed8{/solr,null,AVAILABLE}
   [junit4]   2> 2984300 INFO  (jetty-launcher-2759-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7edb3b73{HTTP/1.1,[http/1.1]}{127.0.0.1:62774}
   [junit4]   2> 2984300 INFO  (jetty-launcher-2759-thread-2) [    ] o.e.j.s.Server Started @2987677ms
   [junit4]   2> 2984300 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62774}
   [junit4]   2> 2984300 ERROR (jetty-launcher-2759-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@64ddcfb0{/solr,null,AVAILABLE}
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:41.555Z
   [junit4]   2> 2984301 INFO  (jetty-launcher-2759-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70ca466a{/solr,null,AVAILABLE}
   [junit4]   2> 2984303 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2984307 INFO  (jetty-launcher-2759-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@127b5aa4{HTTP/1.1,[http/1.1]}{127.0.0.1:40228}
   [junit4]   2> 2984307 INFO  (jetty-launcher-2759-thread-5) [    ] o.e.j.s.Server Started @2987683ms
   [junit4]   2> 2984307 INFO  (jetty-launcher-2759-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@35e78978{HTTP/1.1,[http/1.1]}{127.0.0.1:54199}
   [junit4]   2> 2984307 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40228}
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.e.j.s.Server Started @2987686ms
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54199}
   [junit4]   2> 2984310 ERROR (jetty-launcher-2759-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2984310 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:41.564Z
   [junit4]   2> 2984311 INFO  (jetty-launcher-2759-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7874c37e{/solr,null,AVAILABLE}
   [junit4]   2> 2984311 INFO  (jetty-launcher-2759-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@272d20cb{HTTP/1.1,[http/1.1]}{127.0.0.1:46081}
   [junit4]   2> 2984311 INFO  (jetty-launcher-2759-thread-4) [    ] o.e.j.s.Server Started @2987688ms
   [junit4]   2> 2984311 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46081}
   [junit4]   2> 2984312 ERROR (jetty-launcher-2759-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2984312 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2984312 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2984312 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2984312 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:41.566Z
   [junit4]   2> 2984316 ERROR (jetty-launcher-2759-thread-5) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2984316 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2984316 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2984316 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2984316 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:41.570Z
   [junit4]   2> 2984319 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2984320 INFO  (jetty-launcher-2759-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53839/solr
   [junit4]   2> 2984328 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2984328 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2984329 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2984336 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b0005, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2984336 INFO  (jetty-launcher-2759-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53839/solr
   [junit4]   2> 2984336 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b0002, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2984339 INFO  (jetty-launcher-2759-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53839/solr
   [junit4]   2> 2984340 INFO  (jetty-launcher-2759-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53839/solr
   [junit4]   2> 2984341 INFO  (jetty-launcher-2759-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53839/solr
   [junit4]   2> 2984345 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b000a, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2984543 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2984544 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46081_solr
   [junit4]   2> 2984545 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.c.Overseer Overseer (id=98638696376107022-127.0.0.1:46081_solr-n_0000000000) starting
   [junit4]   2> 2984551 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2984553 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62774_solr
   [junit4]   2> 2984554 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46081_solr
   [junit4]   2> 2984559 INFO  (OverseerStateUpdate-98638696376107022-127.0.0.1:46081_solr-n_0000000000) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2984563 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2984564 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2984565 INFO  (zkCallback-2784-thread-1-processing-n:127.0.0.1:54199_solr) [n:127.0.0.1:54199_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2984565 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2984567 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54199_solr
   [junit4]   2> 2984568 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2984570 INFO  (zkCallback-2784-thread-1-processing-n:127.0.0.1:54199_solr) [n:127.0.0.1:54199_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2984571 INFO  (zkCallback-2785-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2984588 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2984633 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54199.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984637 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46081.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984640 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2984640 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54199.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984640 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54199.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984641 INFO  (jetty-launcher-2759-thread-1) [n:127.0.0.1:54199_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001/node1/.
   [junit4]   2> 2984644 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35734_solr
   [junit4]   2> 2984645 INFO  (zkCallback-2785-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2984646 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2984648 INFO  (zkCallback-2773-thread-1-processing-n:127.0.0.1:35734_solr) [n:127.0.0.1:35734_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2984649 INFO  (zkCallback-2784-thread-1-processing-n:127.0.0.1:54199_solr) [n:127.0.0.1:54199_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2984651 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46081.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984652 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46081.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984652 INFO  (jetty-launcher-2759-thread-4) [n:127.0.0.1:46081_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001/node4/.
   [junit4]   2> 2984677 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2984680 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2984682 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40228_solr
   [junit4]   2> 2984685 INFO  (zkCallback-2785-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2984700 INFO  (zkCallback-2786-thread-1-processing-n:127.0.0.1:40228_solr) [n:127.0.0.1:40228_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2984705 INFO  (zkCallback-2784-thread-1-processing-n:127.0.0.1:54199_solr) [n:127.0.0.1:54199_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2984705 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2984707 INFO  (zkCallback-2773-thread-1-processing-n:127.0.0.1:35734_solr) [n:127.0.0.1:35734_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2984730 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62774.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984730 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35734.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984744 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35734.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984744 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35734.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984745 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62774.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984745 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62774.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984745 INFO  (jetty-launcher-2759-thread-3) [n:127.0.0.1:35734_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001/node3/.
   [junit4]   2> 2984746 INFO  (jetty-launcher-2759-thread-2) [n:127.0.0.1:62774_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001/node2/.
   [junit4]   2> 2984800 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40228.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984805 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40228.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984805 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40228.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2984806 INFO  (jetty-launcher-2759-thread-5) [n:127.0.0.1:40228_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-001/node5/.
   [junit4]   2> 2984835 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b0010, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2984839 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 2984840 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53839/solr ready
   [junit4]   2> 2984844 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCollectionCreateWithoutCoresThenDelete
   [junit4]   2> 2984852 INFO  (qtp2114583954-22076) [n:127.0.0.1:54199_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=2030f0dc-f6ca-46a0-83b6-18940e176c76&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testSolrCloudCollectionWithoutCores&nrtReplicas=2&action=CREATE&numShards=2&createNodeSet=EMPTY&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2984854 INFO  (qtp2114583954-22076) [n:127.0.0.1:54199_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=2030f0dc-f6ca-46a0-83b6-18940e176c76&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testSolrCloudCollectionWithoutCores&nrtReplicas=2&action=CREATE&numShards=2&createNodeSet=EMPTY&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 2984855 INFO  (qtp2114583954-22072) [n:127.0.0.1:54199_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2984857 INFO  (qtp2114583954-22072) [n:127.0.0.1:54199_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 2984858 INFO  (OverseerThreadFactory-8553-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.CreateCollectionCmd Create collection testSolrCloudCollectionWithoutCores
   [junit4]   2> 2984858 WARN  (OverseerThreadFactory-8553-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (testSolrCloudCollectionWithoutCores) without cores.
   [junit4]   2> 2984973 INFO  (OverseerThreadFactory-8553-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2985861 INFO  (qtp2114583954-22078) [n:127.0.0.1:54199_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2985863 INFO  (qtp2114583954-22078) [n:127.0.0.1:54199_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 2985864 INFO  (qtp2114583954-22079) [n:127.0.0.1:54199_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletestatus with params requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=DELETESTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2985866 INFO  (qtp2114583954-22079) [n:127.0.0.1:54199_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=2030f0dc-f6ca-46a0-83b6-18940e176c76&action=DELETESTATUS&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2985866 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testSolrCloudCollectionWithoutCores failOnTimeout:true timeout (sec):330
   [junit4]   1> -
   [junit4]   1> no one is recoverying
   [junit4]   2> 2985867 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testSolrCloudCollectionWithoutCores
   [junit4]   2> 2985869 INFO  (qtp2114583954-22080) [n:127.0.0.1:54199_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testSolrCloudCollectionWithoutCores&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2985876 INFO  (OverseerCollectionConfigSetProcessor-98638696376107022-127.0.0.1:46081_solr-n_0000000000) [n:127.0.0.1:46081_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2985877 INFO  (OverseerThreadFactory-8553-thread-2-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 2986588 INFO  (qtp2114583954-22080) [n:127.0.0.1:54199_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testSolrCloudCollectionWithoutCores&action=DELETE&wt=javabin&version=2} status=0 QTime=719
   [junit4]   2> 2986589 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for collection to disappear - collection: testSolrCloudCollectionWithoutCores failOnTimeout:true timeout (sec):330
   [junit4]   1> -
   [junit4]   2> 2986589 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.AbstractDistribZkTestBase Collection has disappeared - collection: testSolrCloudCollectionWithoutCores
   [junit4]   2> 2986590 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b0011, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2986595 INFO  (jetty-closer-2760-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@35e78978{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2986595 INFO  (jetty-closer-2760-thread-4) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7edb3b73{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2986595 INFO  (jetty-closer-2760-thread-3) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@10d5a0ca{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2986595 INFO  (jetty-closer-2760-thread-5) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@127b5aa4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2986601 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=271361870
   [junit4]   2> 2986605 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=650959765
   [junit4]   2> 2986605 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=256205209
   [junit4]   2> 2986601 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=806556288
   [junit4]   2> 2986606 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2986600 INFO  (jetty-closer-2760-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@272d20cb{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2986606 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5be0bebb: rootName = solr_35734, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7c276db0
   [junit4]   2> 2986606 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2986612 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@23b76133: rootName = solr_62774, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@67f04e9e
   [junit4]   2> 2986610 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2986612 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@71f76a1e: rootName = solr_54199, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@2d7caeed
   [junit4]   2> 2986610 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=750646435
   [junit4]   2> 2986612 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2986614 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2986614 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3ba0f22f: rootName = solr_40228, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@46e979e6
   [junit4]   2> 2986614 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@58ad6a54: rootName = solr_46081, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@4269e0ce
   [junit4]   2> 2986687 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2986687 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5a1a822b: rootName = solr_35734, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2986688 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2986688 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@32e92fcf: rootName = solr_62774, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2986689 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2986689 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@40cda831: rootName = solr_54199, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2986699 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2986699 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4de73893: rootName = solr_40228, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2986700 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2986700 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@63b7e976: rootName = solr_46081, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2986735 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2986736 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5b7bb8ae: rootName = solr_54199, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2986736 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2986736 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1ce4fee3: rootName = solr_35734, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2986740 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2986750 INFO  (jetty-closer-2760-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2986750 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2986750 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2986750 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48442f8e: rootName = solr_46081, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2986750 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@738353c3: rootName = solr_40228, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2986751 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2986751 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@27800b27: rootName = solr_62774, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2986751 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2986751 INFO  (jetty-closer-2760-thread-3) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2986753 WARN  (zkCallback-2784-thread-1-processing-n:127.0.0.1:54199_solr) [n:127.0.0.1:54199_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2986753 WARN  (zkCallback-2773-thread-2-processing-n:127.0.0.1:35734_solr) [n:127.0.0.1:35734_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2986753 INFO  (zkCallback-2785-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 2986754 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 2986754 INFO  (jetty-closer-2760-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@64ddcfb0{/solr,null,UNAVAILABLE}
   [junit4]   2> 2986755 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2986755 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2986755 INFO  (jetty-closer-2760-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98638696376107022-127.0.0.1:46081_solr-n_0000000000) closing
   [junit4]   2> 2986755 INFO  (OverseerStateUpdate-98638696376107022-127.0.0.1:46081_solr-n_0000000000) [n:127.0.0.1:46081_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46081_solr
   [junit4]   2> 2986755 INFO  (zkCallback-2786-thread-1-processing-n:127.0.0.1:40228_solr) [n:127.0.0.1:40228_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (3)
   [junit4]   2> 2986756 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2986756 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2986757 INFO  (jetty-closer-2760-thread-4) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2986757 INFO  (jetty-closer-2760-thread-5) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2986758 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f2a9b000f, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2986759 WARN  (zkCallback-2786-thread-2-processing-n:127.0.0.1:40228_solr) [n:127.0.0.1:40228_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2986760 INFO  (jetty-closer-2760-thread-5) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@70ca466a{/solr,null,UNAVAILABLE}
   [junit4]   2> 2986761 INFO  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
   [junit4]   2> 2988253 INFO  (jetty-closer-2760-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5f1474ac{/solr,null,UNAVAILABLE}
   [junit4]   2> 2988260 WARN  (zkCallback-2785-thread-1-processing-n:127.0.0.1:46081_solr) [n:127.0.0.1:46081_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2988261 INFO  (jetty-closer-2760-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7874c37e{/solr,null,UNAVAILABLE}
   [junit4]   2> 2988262 WARN  (zkCallback-2783-thread-1-processing-n:127.0.0.1:62774_solr) [n:127.0.0.1:62774_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2988263 INFO  (jetty-closer-2760-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@35c16ed8{/solr,null,UNAVAILABLE}
   [junit4]   2> 2988264 ERROR (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2988268 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53839 53839
   [junit4]   2> 2988310 INFO  (Thread-4608) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53839 53839
   [junit4]   2> 2988323 WARN  (Thread-4608) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/aliases.json
   [junit4]   2> 6 /solr/clusterprops.json
   [junit4]   2> 5 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/clusterstate.json
   [junit4]   2> 2 /solr/overseer_elect/election/98638696376107020-127.0.0.1:62774_solr-n_0000000001
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/live_nodes
   [junit4]   2> 6 /solr/collections
   [junit4]   2>
   [junit4]   2> 2988324 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCollectionCreateWithoutCoresThenDelete
   [junit4]   2> 2988339 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002
   [junit4]   2> 2988339 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2988339 INFO  (Thread-4657) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2988339 INFO  (Thread-4657) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2988344 ERROR (Thread-4657) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2988440 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer start zk server on port:42955
   [junit4]   2> 2988475 INFO  (jetty-launcher-2796-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2988475 INFO  (jetty-launcher-2796-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2988475 INFO  (jetty-launcher-2796-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2988476 INFO  (jetty-launcher-2796-thread-5) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2988476 INFO  (jetty-launcher-2796-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2988479 INFO  (jetty-launcher-2796-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e5ee47{/solr,null,AVAILABLE}
   [junit4]   2> 2988479 INFO  (jetty-launcher-2796-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2dbc2a8d{/solr,null,AVAILABLE}
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@271edb1e{HTTP/1.1,[http/1.1]}{127.0.0.1:48038}
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-1) [    ] o.e.j.s.Server Started @2991859ms
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48038}
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@43fdcafc{HTTP/1.1,[http/1.1]}{127.0.0.1:42315}
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@72a512e6{/solr,null,AVAILABLE}
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-3) [    ] o.e.j.s.Server Started @2991859ms
   [junit4]   2> 2988481 ERROR (jetty-launcher-2796-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@119db81b{HTTP/1.1,[http/1.1]}{127.0.0.1:39039}
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-4) [    ] o.e.j.s.Server Started @2991860ms
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39039}
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2988481 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42315}
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:45.736Z
   [junit4]   2> 2988482 ERROR (jetty-launcher-2796-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2988482 ERROR (jetty-launcher-2796-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:45.736Z
   [junit4]   2> 2988482 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2988483 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2988483 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2988483 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:45.737Z
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@432f939c{/solr,null,AVAILABLE}
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@44acfa43{HTTP/1.1,[http/1.1]}{127.0.0.1:57285}
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.e.j.s.Server Started @2991867ms
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57285}
   [junit4]   2> 2988489 ERROR (jetty-launcher-2796-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a5e8f87{/solr,null,AVAILABLE}
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2988489 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:45.744Z
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6910878a{HTTP/1.1,[http/1.1]}{127.0.0.1:63275}
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.e.j.s.Server Started @2991868ms
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63275}
   [junit4]   2> 2988490 ERROR (jetty-launcher-2796-thread-5) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2988490 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:45.744Z
   [junit4]   2> 2988492 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2988494 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2988501 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2988502 INFO  (jetty-launcher-2796-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2988504 INFO  (jetty-launcher-2796-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2988505 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2988506 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e6f5f3afe0004, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2988507 INFO  (jetty-launcher-2796-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2988508 INFO  (jetty-launcher-2796-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2988512 INFO  (jetty-launcher-2796-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2988674 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2988675 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57285_solr
   [junit4]   2> 2988677 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.c.Overseer Overseer (id=98638696651030539-127.0.0.1:57285_solr-n_0000000000) starting
   [junit4]   2> 2988684 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2988686 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63275_solr
   [junit4]   2> 2988686 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2988687 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2988692 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2988695 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2988695 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2988698 INFO  (zkCallback-2823-thread-1-processing-n:127.0.0.1:42315_solr) [n:127.0.0.1:42315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2988700 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39039_solr
   [junit4]   2> 2988702 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57285_solr
   [junit4]   2> 2988702 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42315_solr
   [junit4]   2> 2988704 INFO  (zkCallback-2823-thread-1-processing-n:127.0.0.1:42315_solr) [n:127.0.0.1:42315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2988704 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2988704 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2988704 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2988706 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2988717 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 2988721 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 2988721 INFO  (zkCallback-2823-thread-1-processing-n:127.0.0.1:42315_solr) [n:127.0.0.1:42315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 2988734 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2988805 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42315.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988818 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42315.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988818 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42315.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988819 INFO  (jetty-launcher-2796-thread-3) [n:127.0.0.1:42315_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node3/.
   [junit4]   2> 2988851 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57285.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988859 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57285.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988859 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57285.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988860 INFO  (jetty-launcher-2796-thread-2) [n:127.0.0.1:57285_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node2/.
   [junit4]   2> 2988872 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2988875 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63275.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988880 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63275.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988880 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63275.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988881 INFO  (jetty-launcher-2796-thread-5) [n:127.0.0.1:63275_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node5/.
   [junit4]   2> 2988882 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2988884 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48038_solr
   [junit4]   2> 2988887 INFO  (zkCallback-2823-thread-1-processing-n:127.0.0.1:42315_solr) [n:127.0.0.1:42315_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2988887 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2988890 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2988893 INFO  (zkCallback-2810-thread-1-processing-n:127.0.0.1:48038_solr) [n:127.0.0.1:48038_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2988907 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2988947 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39039.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988951 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39039.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988951 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39039.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2988952 INFO  (jetty-launcher-2796-thread-4) [n:127.0.0.1:39039_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node4/.
   [junit4]   2> 2989006 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48038.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2989012 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48038.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2989012 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48038.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2989013 INFO  (jetty-launcher-2796-thread-1) [n:127.0.0.1:48038_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node1/.
   [junit4]   2> 2989048 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 2989049 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42955/solr ready
   [junit4]   2> 2989054 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCollectionCreateSearchDelete
   [junit4]   2> 2989057 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@43fdcafc{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2989059 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=595759710
   [junit4]   2> 2989059 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2989059 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1f94471: rootName = solr_42315, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@75dd3f6a
   [junit4]   2> 2989064 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2989064 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4bceef17: rootName = solr_42315, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@29153079
   [junit4]   2> 2989067 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2989067 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@769afb65: rootName = solr_42315, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@10737e81
   [junit4]   2> 2989069 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2989069 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2989071 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (4)
   [junit4]   2> 2989071 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (4)
   [junit4]   2> 2989074 INFO  (zkCallback-2832-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (4)
   [junit4]   2> 2989074 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (4)
   [junit4]   2> 2989074 INFO  (zkCallback-2810-thread-1-processing-n:127.0.0.1:48038_solr) [n:127.0.0.1:48038_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (4)
   [junit4]   2> 2990576 WARN  (zkCallback-2823-thread-1-processing-n:127.0.0.1:42315_solr) [n:127.0.0.1:42315_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2990577 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@e5ee47{/solr,null,UNAVAILABLE}
   [junit4]   2> 2990579 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2990580 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24876f24{/solr,null,AVAILABLE}
   [junit4]   2> 2990580 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3d3f06ac{HTTP/1.1,[http/1.1]}{127.0.0.1:44502}
   [junit4]   2> 2990580 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.e.j.s.Server Started @2993959ms
   [junit4]   2> 2990580 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44502}
   [junit4]   2> 2990583 ERROR (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2990584 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2990584 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2990584 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2990584 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-11T05:17:47.838Z
   [junit4]   2> 2990587 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2990594 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42955/solr
   [junit4]   2> 2990607 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2990610 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2990613 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44502_solr
   [junit4]   2> 2990615 INFO  (zkCallback-2819-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990615 INFO  (zkCallback-2817-thread-1-processing-n:127.0.0.1:63275_solr) [n:127.0.0.1:63275_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990615 INFO  (zkCallback-2810-thread-2-processing-n:127.0.0.1:48038_solr) [n:127.0.0.1:48038_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990616 INFO  (zkCallback-2822-thread-1-processing-n:127.0.0.1:39039_solr) [n:127.0.0.1:39039_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990616 INFO  (zkCallback-2832-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990619 INFO  (zkCallback-2837-thread-1-processing-n:127.0.0.1:44502_solr) [n:127.0.0.1:44502_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2990757 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44502.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2990763 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44502.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2990763 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44502.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b807f26
   [junit4]   2> 2990764 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete-seed#[24E1A3E1F5AF5EDC]) [n:127.0.0.1:44502_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001/tempDir-002/node6/.
   [junit4]   2> 2990808 INFO  (qtp507658215-22229) [n:127.0.0.1:63275_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=e619e133-94de-49d8-9b20-316226a07134&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testcollection&nrtReplicas=2&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2990810 INFO  (qtp507658215-22229) [n:127.0.0.1:63275_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=e619e133-94de-49d8-9b20-316226a07134&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testcollection&nrtReplicas=2&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 2990811 INFO  (qtp507658215-22239) [n:127.0.0.1:63275_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=e619e133-94de-49d8-9b20-316226a07134&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2990813 INFO  (qtp507658215-22239) [n:127.0.0.1:63275_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=e619e133-94de-49d8-9b20-316226a07134&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 2990814 INFO  (OverseerThreadFactory-8590-thread-1-processing-n:127.0.0.1:57285_solr) [n:127.0.0.1:57285_solr    ] o.a.s.c.CreateCollectionCmd Create collection testcollection
   [junit4]   2> 2990930 INFO  (OverseerStateUpdate-98638696651030539-127.0.0.1:57285_solr-n_0000000000) [n:127.0.0.1:57285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testcollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testcollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:39039/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 2990937 INFO  (OverseerStateUpdate-98638696651030539-127.0.0.1:57285_solr-n_0000000000) [n:127.0.0.1:57285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testcollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testcollection_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:48038/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 2990942 INFO  (OverseerStateUpdate-98638696651030539-127.0.0.1:57285_solr-n_0000000000) [n:127.0.0.1:57285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testcollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testcollection_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57285/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 2990948 INFO  (OverseerStateUpdate-98638696651030539-127.0.0.1:57285_solr-n_0000000000) [n:127.0.0.1:57285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testcollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testcollection_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:63275/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 2991155 INFO  (qtp31853306-22218) [n:127.0.0.1:39039_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testcollection&version=2&replicaType=NRT&async=e619e133-94de-49d8-9b20-316226a071342121454432381152&coreNodeName=core_node3&name=testcollection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=0
   [junit4]   2> 2991160 INFO  (parallelCoreAdminExecutor-8597-thread-1-processing-n:127.0.0.1:39039_solr e619e133-94de-49d8-9b20-316226a071342121454432381152 CREATE) [n:127.0.0.1:39039_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testcollection&version=2&replicaType=NRT&async=e619e133-94de-49d8-9b20-316226

[...truncated too long message...]

ang.Thread.run(Thread.java:748)
   [junit4]   2> 3021624 INFO  (jetty-closer-2846-thread-6) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7cdd02ba{/solr,null,UNAVAILABLE}
   [junit4]   2> 3021632 INFO  (coreCloseExecutor-8771-thread-1) [n:127.0.0.1:64926_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testStopAllStartAllCollection.shard1.leader, tag=732626107
   [junit4]   2> 3021636 INFO  (jetty-closer-2846-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1ff84b33{/solr,null,UNAVAILABLE}
   [junit4]   2> 3023110 WARN  (zkCallback-2892-thread-2-processing-n:127.0.0.1:53246_solr) [n:127.0.0.1:53246_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 3023111 INFO  (jetty-closer-2846-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@19bcdcea{/solr,null,UNAVAILABLE}
   [junit4]   2> 3023116 ERROR (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3023121 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45345 45345
   [junit4]   2> 3023177 INFO  (Thread-4725) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45345 45345
   [junit4]   2> 3023196 WARN  (Thread-4725) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 10 /solr/aliases.json
   [junit4]   2> 10 /solr/clusterprops.json
   [junit4]   2> 9 /solr/security.json
   [junit4]   2> 8 /solr/configs/solrCloudCollectionConfig
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 76 /solr/collections/testStopAllStartAllCollection/state.json
   [junit4]   2> 10 /solr/clusterstate.json
   [junit4]   2> 5 /solr/overseer_elect/election/98638697536487438-127.0.0.1:39010_solr-n_0000000000
   [junit4]   2> 2 /solr/overseer_elect/election/98638697536487444-127.0.0.1:50250_solr-n_0000000005
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 10 /solr/collections
   [junit4]   2> 6 /solr/live_nodes
   [junit4]   2>
   [junit4]   2> 3023196 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[24E1A3E1F5AF5EDC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStopAllStartAll
   [junit4]   2> 3023198 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[24E1A3E1F5AF5EDC]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45345 45345
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCollectionsAPIViaSolrCloudCluster_24E1A3E1F5AF5EDC-001
   [junit4]   2> Sep 11, 2017 5:18:20 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, maxPointsInLeafNode=1692, maxMBSortInHeap=5.816716531473446, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=America/Jujuy
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=161466856,total=516030464
   [junit4]   2> NOTE: All tests run in this JVM: [TestChildDocTransformer, ZkSolrClientTest, TestFieldCacheWithThreads, TestMultiWordSynonyms, TestPayloadScoreQParserPlugin, TestPartialUpdateDeduplication, TestLegacyField, TestMacroExpander, BaseCdcrDistributedZkTest, BasicZkTest, EnumFieldTest, LeaderInitiatedRecoveryOnCommitTest, TestRebalanceLeaders, ChangedSchemaMergeTest, TestAnalyzeInfixSuggestions, TestStressUserVersions, TestRandomCollapseQParserPlugin, TestCoreDiscovery, TestFieldResource, MoreLikeThisHandlerTest, BadCopyFieldTest, TestRandomFlRTGCloud, TestRangeQuery, TestSlowCompositeReaderWrapper, JsonLoaderTest, TestShortCircuitedRequests, TestFieldCacheReopen, ZkStateWriterTest, ReplicationFactorTest, TestGraphTermsQParserPlugin, SuggesterTSTTest, SolrMetricsIntegrationTest, TestNumericTerms32, TestRealTimeGet, TestSimpleQParserPlugin, TestReload, AtomicUpdatesTest, TestLMJelinekMercerSimilarityFactory, TestCollationField, PreAnalyzedFieldTest, ExitableDirectoryReaderTest, OpenExchangeRatesOrgProviderTest, SignatureUpdateProcessorFactoryTest, TestLeaderInitiatedRecoveryThread, DistributedFacetPivotSmallTest, TestSolrCloudSnapshots, TestSystemIdResolver, BadIndexSchemaTest, DateRangeFieldTest, CdcrRequestHandlerTest, TestPushWriter, TestSearchPerf, TestDocumentBuilder, TestCloudRecovery, CollectionStateFormat2Test, TestCollationFieldDocValues, FieldMutatingUpdateProcessorTest, UpdateParamsTest, BadComponentTest, TestSolrIndexConfig, HdfsDirectoryTest, TestRequestForwarding, PluginInfoTest, TestCrossCoreJoin, TestQueryTypes, TestReloadDeadlock, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestBM25SimilarityFactory, ConnectionReuseTest, TestDynamicFieldResource, TestInfoStreamLogging, TestPerFieldSimilarity, TestRestManager, TestRecovery, TestElisionMultitermQuery, AutoCommitTest, TestSubQueryTransformerCrossCore, TestExportWriter, DOMUtilTest, TestLRUCache, TestManagedResource, CacheHeaderTest, BasicFunctionalityTest, SolrCoreCheckLockOnStartupTest, TestCharFilters, CollectionsAPIAsyncDistributedZkTest, TestStressInPlaceUpdates, TestUseDocValuesAsStored, ExternalFileFieldSortTest, TestDynamicLoading, TestTolerantUpdateProcessorRandomCloud, TestSuggestSpellingConverter, TestWordDelimiterFilterFactory, TestCorePropertiesReload, OverriddenZkACLAndCredentialsProvidersTest, QueryEqualityTest, BigEndianAscendingWordDeserializerTest, MigrateRouteKeyTest, PreAnalyzedFieldManagedSchemaCloudTest, SystemInfoHandlerTest, TestClusterStateMutator, ShardRoutingCustomTest, HttpPartitionTest, TestManagedSchemaThreadSafety, TestConfigSetProperties, BlobRepositoryCloudTest, PolyFieldTest, DocumentAnalysisRequestHandlerTest, TestSolrCloudWithDelegationTokens, SOLR749Test, SolrCloudReportersTest, TestCloudNestedDocsSort, CursorMarkTest, DeleteReplicaTest, TestConfig, TestExactSharedStatsCache, TestRawTransformer, HdfsRecoveryZkTest, TestLRUStatsCache, TestHdfsBackupRestoreCore, TestEmbeddedSolrServerSchemaAPI, TestDelegationWithHadoopAuth, FullSolrCloudDistribCmdsTest, V2StandaloneTest, TestStressLucene, TestManagedSynonymFilterFactory, SpellPossibilityIteratorTest, TestSolrConfigHandler, TestDocSet, DistribDocExpirationUpdateProcessorTest, DeleteStatusTest, LoggingHandlerTest, DocValuesNotIndexedTest, TestReRankQParserPlugin, OverseerStatusTest, CollectionsAPIDistributedZkTest, TestDistributedMissingSort, SolrIndexMetricsTest, EchoParamsTest, TestSolrFieldCacheBean, TestFileDictionaryLookup, UpdateLogTest, TestPullReplica, TestCloudInspectUtil, TestHighFrequencyDictionaryFactory, WrapperMergePolicyFactoryTest, TestUniqueKeyFieldResource, SolrIndexConfigTest, TestXmlQParser, MBeansHandlerTest, TestTrackingShardHandlerFactory, HighlighterMaxOffsetTest, InfixSuggestersTest, TestRuleBasedAuthorizationPlugin, HdfsBasicDistributedZkTest, TestExceedMaxTermLength, TestJoin, TriLevelCompositeIdRoutingTest, PingRequestHandlerTest, TestConfigsApi, ShowFileRequestHandlerTest, DebugComponentTest, TestExclusionRuleCollectionAccess, BlockJoinFacetSimpleTest, SortSpecParsingTest, NodeMutatorTest, DeleteLastCustomShardedReplicaTest, TestFieldCacheSortRandom, PropertiesRequestHandlerTest, SchemaVersionSpecificBehaviorTest, RollingRestartTest, MoveReplicaHDFSTest, CdcrReplicationDistributedZkTest, ShufflingReplicaListTransformerTest, SuggestComponentContextFilterQueryTest, TestAnalyzedSuggestions, TestBlobHandler, TestAuthenticationFramework, BasicDistributedZk2Test, BasicDistributedZkTest, SimpleFacetsTest, StatsComponentTest, QueryElevationComponentTest, ConvertedLegacyTest, TestSort, TestFiltering, TestLazyCores, TestBadConfig, DistributedQueryElevationComponentTest, SpellCheckCollatorTest, CoreAdminHandlerTest, WordBreakSolrSpellCheckerTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, TermsComponentTest, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, TestCSVResponseWriter, TestPHPSerializedResponseWriter, TestComponentsName, TestLFUCache, HighlighterConfigTest, TestQuerySenderListener, AlternateDirectoryTest, CopyFieldTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestFuzzyAnalyzedSuggestions, TestLuceneMatchVersion, TestSweetSpotSimilarityFactory, TestFastWriter, TestFastOutputStream, TestFastLRUCache, DateMathParserTest, PrimUtilsTest, SpellingQueryConverterTest, TestSolrJ, ZkNodePropsTest, FileUtilsTest, TestRTGBase, CursorPagingTest, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, AssignTest, CdcrVersionReplicationTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CustomCollectionTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, MoveReplicaTest, RecoveryAfterSoftCommitTest, SolrCloudExampleTest, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster]
   [junit4] Completed [528/731 (1!)] on J1 in 39.91s, 3 tests, 1 error <<< FAILURES!

[...truncated 46987 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 170 - Failure!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/170/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 9398 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/build.xml:770: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/build.xml:495: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:2256: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/module-build.xml:58: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/module-build.xml:55: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:831: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:843: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:2044: Compile failed; see the compiler error output for details.

Total time: 16 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 171 - Still unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/171/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

7 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSTest

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64369 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64369 within 30000 ms
        at __randomizedtesting.SeedInfo.seed([6A10D1E01D2864EB]:0)
        at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
        at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
        at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
        at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
        at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
        at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
        at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
        at org.apache.solr.cloud.MoveReplicaTest.setupCluster(MoveReplicaTest.java:53)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64369 within 30000 ms
        at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
        at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
        ... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSTest

Error Message:


Stack Trace:
java.lang.NullPointerException
        at __randomizedtesting.SeedInfo.seed([6A10D1E01D2864EB]:0)
        at org.apache.solr.cloud.MoveReplicaHDFSTest.teardownClass(MoveReplicaHDFSTest.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSTest

Error Message:
54 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSTest:     1) Thread[id=22670, name=qtp1468910649-22670, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=22703, name=qtp550049991-22703, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=22667, name=qtp1468910649-22667, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=22702, name=org.eclipse.jetty.server.session.HashSessionManager@3933cd4dTimer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=22694, name=qtp781667853-22694, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=22682, name=Scheduler-1418452177, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=22696, name=qtp781667853-22696, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=22706, name=qtp781667853-22706, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=22664, name=org.eclipse.jetty.server.session.HashSessionManager@22debd8aTimer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=22710, name=qtp1468910649-22710, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=22652, name=jetty-launcher-3360-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=22659, name=qtp1514945000-22659, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=22671, name=qtp1468910649-22671-acceptor-0@772231b4-ServerConnector@15c588a4{HTTP/1.1,[http/1.1]}{127.0.0.1:46663}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=22663, name=qtp1514945000-22663, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=22647, name=SyncThread:0, state=WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)   16) Thread[id=22661, name=qtp1514945000-22661, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=22691, name=org.eclipse.jetty.server.session.HashSessionManager@5d825de9Timer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=22673, name=org.eclipse.jetty.server.session.HashSessionManager@5345d214Timer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=22654, name=jetty-launcher-3360-thread-3, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=22695, name=qtp781667853-22695-acceptor-0@12e191cb-ServerConnector@1ee90fd9{HTTP/1.1,[http/1.1]}{127.0.0.1:45062}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=22700, name=qtp781667853-22700, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=22704, name=qtp550049991-22704, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=22657, name=qtp1514945000-22657, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=22660, name=qtp1514945000-22660, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=22648, name=ProcessThread(sid:0 cport:64369):, state=WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   26) Thread[id=22677, name=qtp1514945000-22677, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=22658, name=qtp1514945000-22658-acceptor-0@4fc9e5d8-ServerConnector@42bffcf4{HTTP/1.1,[http/1.1]}{127.0.0.1:64369}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=22668, name=qtp1468910649-22668, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=22653, name=jetty-launcher-3360-thread-2, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=22655, name=jetty-launcher-3360-thread-4, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=22662, name=qtp1514945000-22662, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=22685, name=qtp550049991-22685, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   33) Thread[id=22687, name=qtp550049991-22687, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=22705, name=qtp781667853-22705, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=22676, name=qtp1468910649-22676, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=22690, name=qtp550049991-22690, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   37) Thread[id=22683, name=qtp550049991-22683, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   38) Thread[id=22646, name=SessionTracker, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   39) Thread[id=22644, name=Thread-4658, state=WAITING, group=TGRP-MoveReplicaHDFSTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)   40) Thread[id=22701, name=qtp781667853-22701, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   41) Thread[id=22666, name=qtp1468910649-22666, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   42) Thread[id=22697, name=qtp781667853-22697, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   43) Thread[id=22645, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)   44) Thread[id=22672, name=qtp1468910649-22672, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   45) Thread[id=22669, name=qtp1468910649-22669, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   46) Thread[id=22688, name=qtp550049991-22688, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   47) Thread[id=22684, name=qtp550049991-22684, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   48) Thread[id=22686, name=qtp550049991-22686-acceptor-0@5fd3672a-ServerConnector@17a19a24{HTTP/1.1,[http/1.1]}{127.0.0.1:56636}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   49) Thread[id=22711, name=qtp781667853-22711, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   50) Thread[id=22681, name=qtp1514945000-22681, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   51) Thread[id=22712, name=qtp550049991-22712, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   52) Thread[id=22656, name=qtp1514945000-22656, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   53) Thread[id=22699, name=qtp781667853-22699, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   54) Thread[id=22707, name=qtp1468910649-22707, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 54 threads leaked from SUITE scope at org.apache.solr.cloud.MoveReplicaHDFSTest:
   1) Thread[id=22670, name=qtp1468910649-22670, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=22703, name=qtp550049991-22703, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=22667, name=qtp1468910649-22667, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=22702, name=org.eclipse.jetty.server.session.HashSessionManager@3933cd4dTimer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=22694, name=qtp781667853-22694, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=22682, name=Scheduler-1418452177, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=22696, name=qtp781667853-22696, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=22706, name=qtp781667853-22706, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=22664, name=org.eclipse.jetty.server.session.HashSessionManager@22debd8aTimer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=22710, name=qtp1468910649-22710, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=22652, name=jetty-launcher-3360-thread-1, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=22659, name=qtp1514945000-22659, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=22671, name=qtp1468910649-22671-acceptor-0@772231b4-ServerConnector@15c588a4{HTTP/1.1,[http/1.1]}{127.0.0.1:46663}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=22663, name=qtp1514945000-22663, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=22647, name=SyncThread:0, state=WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
  16) Thread[id=22661, name=qtp1514945000-22661, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=22691, name=org.eclipse.jetty.server.session.HashSessionManager@5d825de9Timer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=22673, name=org.eclipse.jetty.server.session.HashSessionManager@5345d214Timer, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=22654, name=jetty-launcher-3360-thread-3, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=22695, name=qtp781667853-22695-acceptor-0@12e191cb-ServerConnector@1ee90fd9{HTTP/1.1,[http/1.1]}{127.0.0.1:45062}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=22700, name=qtp781667853-22700, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=22704, name=qtp550049991-22704, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=22657, name=qtp1514945000-22657, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=22660, name=qtp1514945000-22660, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=22648, name=ProcessThread(sid:0 cport:64369):, state=WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
  26) Thread[id=22677, name=qtp1514945000-22677, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=22658, name=qtp1514945000-22658-acceptor-0@4fc9e5d8-ServerConnector@42bffcf4{HTTP/1.1,[http/1.1]}{127.0.0.1:64369}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=22668, name=qtp1468910649-22668, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=22653, name=jetty-launcher-3360-thread-2, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=22655, name=jetty-launcher-3360-thread-4, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  31) Thread[id=22662, name=qtp1514945000-22662, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=22685, name=qtp550049991-22685, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  33) Thread[id=22687, name=qtp550049991-22687, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  34) Thread[id=22705, name=qtp781667853-22705, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  35) Thread[id=22676, name=qtp1468910649-22676, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  36) Thread[id=22690, name=qtp550049991-22690, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  37) Thread[id=22683, name=qtp550049991-22683, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  38) Thread[id=22646, name=SessionTracker, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  39) Thread[id=22644, name=Thread-4658, state=WAITING, group=TGRP-MoveReplicaHDFSTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
  40) Thread[id=22701, name=qtp781667853-22701, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  41) Thread[id=22666, name=qtp1468910649-22666, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  42) Thread[id=22697, name=qtp781667853-22697, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  43) Thread[id=22645, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
  44) Thread[id=22672, name=qtp1468910649-22672, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  45) Thread[id=22669, name=qtp1468910649-22669, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  46) Thread[id=22688, name=qtp550049991-22688, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  47) Thread[id=22684, name=qtp550049991-22684, state=TIMED_WAITING, group=TGRP-MoveReplicaHDFSTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  48) Thread[id=22686, name=qtp550049991-22686-acceptor-0@5fd3672a-ServerConnector@17a19a24{HTTP/1.1,[http/1.1]}{127.0.0.1:56636}, state=RUNNABLE, group=TGRP-MoveReplicaHDFSTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept

[...truncated too long message...]

lection1_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 225376 INFO  (zkCallback-986-thread-3-processing-n:127.0.0.1:45284_solr) [n:127.0.0.1:45284_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 225376 INFO  (zkCallback-986-thread-2-processing-n:127.0.0.1:45284_solr) [n:127.0.0.1:45284_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 225390 INFO  (coreCloseExecutor-647-thread-4) [n:127.0.0.1:45284_solr c:destination s:shard1 r:core_node3 x:destination_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.destination.shard1.leader, tag=868584935
   [junit4]   2> 225390 INFO  (coreCloseExecutor-647-thread-3) [n:127.0.0.1:45284_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.mainCorpus.shard1.replica_n1, tag=186700374
   [junit4]   2> 225390 INFO  (coreCloseExecutor-647-thread-3) [n:127.0.0.1:45284_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@771adef6: rootName = solr_45284, domain = solr.core.mainCorpus.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.mainCorpus.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2c22ec5
   [junit4]   2> 225407 INFO  (coreCloseExecutor-647-thread-3) [n:127.0.0.1:45284_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard1.leader, tag=186700374
   [junit4]   2> 225453 INFO  (jetty-closer-973-thread-4) [    ] o.a.s.c.Overseer Overseer (id=98642552625233926-127.0.0.1:45284_solr-n_0000000000) closing
   [junit4]   2> 225453 INFO  (OverseerStateUpdate-98642552625233926-127.0.0.1:45284_solr-n_0000000000) [n:127.0.0.1:45284_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45284_solr
   [junit4]   2> 226639 WARN  (zkCallback-993-thread-1-processing-n:127.0.0.1:48597_solr) [n:127.0.0.1:48597_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 226640 INFO  (jetty-closer-973-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@232eb1bf{/solr,null,UNAVAILABLE}
   [junit4]   2> 226747 WARN  (zkCallback-995-thread-3-processing-n:127.0.0.1:53858_solr) [n:127.0.0.1:53858_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 226748 INFO  (jetty-closer-973-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2cce7de0{/solr,null,UNAVAILABLE}
   [junit4]   2> 226760 WARN  (zkCallback-994-thread-2-processing-n:127.0.0.1:40780_solr) [n:127.0.0.1:40780_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 226761 INFO  (jetty-closer-973-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@44c5d4ff{/solr,null,UNAVAILABLE}
   [junit4]   2> 226966 WARN  (zkCallback-986-thread-3-processing-n:127.0.0.1:45284_solr) [n:127.0.0.1:45284_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 226967 INFO  (jetty-closer-973-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@762839b2{/solr,null,UNAVAILABLE}
   [junit4]   2> 226970 ERROR (SUITE-StreamExpressionTest-seed#[C3D7DA26982934D]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 226973 INFO  (SUITE-StreamExpressionTest-seed#[C3D7DA26982934D]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43764 43764
   [junit4]   2> 238541 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 238542 INFO  (Thread-355) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43764 43764
   [junit4]   2> 238543 WARN  (Thread-355) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 72 /solr/aliases.json
   [junit4]   2> 72 /solr/clusterprops.json
   [junit4]   2> 8 /solr/configs/conf
   [junit4]   2> 4 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 72 /solr/clusterstate.json
   [junit4]   2> 21 /solr/collections/destination/state.json
   [junit4]   2> 21 /solr/collections/mainCorpus/state.json
   [junit4]   2> 20 /solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 20 /solr/collections/destinationCollection/state.json
   [junit4]   2> 19 /solr/collections/workQueue/state.json
   [junit4]   2> 18 /solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 11 /solr/collections/collection1_collection/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98642552625233932-127.0.0.1:53858_solr-n_0000000001
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 72 /solr/live_nodes
   [junit4]   2> 72 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_C3D7DA26982934D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {expr_s=FST50, join2_s=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=LuceneFixedGap), subject=PostingsFormat(name=LuceneFixedGap), a1_s=PostingsFormat(name=LuceneFixedGap), a2_s=PostingsFormat(name=Memory), test_t=PostingsFormat(name=Memory), level1_s=FST50, body_t=PostingsFormat(name=Memory), ident_s=PostingsFormat(name=LuceneFixedGap), s_multi=PostingsFormat(name=Memory), col_s=PostingsFormat(name=LuceneFixedGap), a_ss=PostingsFormat(name=Direct), level2_s=PostingsFormat(name=Direct), a_s=PostingsFormat(name=Memory), a_t=PostingsFormat(name=Direct), side_s=PostingsFormat(name=Memory), id=FST50, text=PostingsFormat(name=Memory)}, docValues:{expr_s=DocValuesFormat(name=Memory), join2_s=DocValuesFormat(name=Lucene70), multiDefault=DocValuesFormat(name=Lucene70), price_f=DocValuesFormat(name=Lucene70), a1_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Asserting), a_f=DocValuesFormat(name=Lucene70), join1_i=DocValuesFormat(name=Memory), a2_s=DocValuesFormat(name=Direct), a_i=DocValuesFormat(name=Memory), join3_i=DocValuesFormat(name=Direct), test_i=DocValuesFormat(name=Lucene70), field_i=DocValuesFormat(name=Lucene70), level1_s=DocValuesFormat(name=Memory), s_multi=DocValuesFormat(name=Direct), col_s=DocValuesFormat(name=Lucene70), a_ss=DocValuesFormat(name=Asserting), level2_s=DocValuesFormat(name=Asserting), a_s=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Asserting), i_multi=DocValuesFormat(name=Memory), b_ls=DocValuesFormat(name=Lucene70), test_dt=DocValuesFormat(name=Memory), ident_s=DocValuesFormat(name=Lucene70), order_i=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Asserting), side_s=DocValuesFormat(name=Direct), miles_i=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1790, maxMBSortInHeap=7.518650305943785, sim=RandomSimilarity(queryNorm=true): {}, locale=sk-SK, timezone=Asia/Amman
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=148206912,total=339738624
   [junit4]   2> NOTE: All tests run in this JVM: [SolrExceptionTest, MergeIndexesEmbeddedTest, NotEvaluatorTest, TestLBHttpSolrClient, NoOpResponseParserTest, LargeVolumeBinaryJettyTest, TestDelegationTokenResponse, JdbcTest, HyperbolicSineEvaluatorTest, StreamingTest, ShardParamsTest, CloudSolrClientBuilderTest, AddEvaluatorTest, TestRetryUtil, NormalDistributionEvaluatorTest, RecursiveEvaluatorTest, CeilingEvaluatorTest, CloudSolrClientMultiConstructorTest, LessThanEvaluatorTest, HttpSolrClientSSLAuthConPoolTest, ConcurrentUpdateSolrClientBuilderTest, ArcSineEvaluatorTest, TestCollectionAdminRequest, TestNamedListCodec, TestPolicy, SelectWithEvaluatorsTest, ExclusiveOrEvaluatorTest, SolrQueryTest, TestJsonRecordReader, ArrayEvaluatorTest, TestCloudSolrClientConnections, GreaterThanEvaluatorTest, StreamExpressionToExpessionTest, HttpSolrClientBuilderTest, ClientUtilsTest, SolrExampleStreamingBinaryTest, SolrExampleXMLTest, SolrExampleEmbeddedTest, TestSolrProperties, TestFastInputStream, DocumentAnalysisResponseTest, TestUpdateRequestCodec, FacetFieldTest, CloudSolrClientCacheTest, HttpSolrClientConPoolTest, GraphTest, JdbcDriverTest, StreamExpressionTest]
   [junit4] Completed [142/145 (1!)] on J1 in 107.01s, 109 tests, 1 error, 9 skipped <<< FAILURES!

[...truncated 45366 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 172 - Still Unstable!

Policeman Jenkins Server-2
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/172/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([F524A849B49803B9:7652F7BB62E10D18]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:184)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12994 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
   [junit4]   2> 3419055 INFO  (SUITE-TestCloudRecovery-seed#[F524A849B49803B9]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_F524A849B49803B9-001/init-core-data-001
   [junit4]   2> 3419056 WARN  (SUITE-TestCloudRecovery-seed#[F524A849B49803B9]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 3419056 INFO  (SUITE-TestCloudRecovery-seed#[F524A849B49803B9]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFiel