[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 401 - Failure

classic Classic list List threaded Threaded
38 messages Options
12
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 401 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/401/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/.git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor595.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/.git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor595.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/.git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor595.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #391 to compare, so performing full copy of artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run?
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 1 day 7 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 402 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/402/

3 tests failed.
FAILED:  org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([31D3822807824943:4A471897854268F4]:0)
        at java.util.Arrays.copyOfRange(Arrays.java:3664)
        at java.lang.String.<init>(String.java:207)
        at java.lang.StringBuilder.toString(StringBuilder.java:407)
        at org.apache.lucene.index.CorruptIndexException.<init>(CorruptIndexException.java:62)
        at org.apache.lucene.index.CorruptIndexException.<init>(CorruptIndexException.java:47)
        at org.apache.lucene.codecs.CodecUtil.checkFooter(CodecUtil.java:471)
        at org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat.read(Lucene70SegmentInfoFormat.java:266)
        at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:361)
        at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:291)
        at org.apache.lucene.index.DirectoryReader.listCommits(DirectoryReader.java:277)
        at org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:233)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)


FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
shard1 is not consistent.  Got 1120 from http://127.0.0.1:41538/je_a/y/collection1_shard1_replica_n5 (previous client) and got 4 from http://127.0.0.1:46233/je_a/y/collection1_shard1_replica_n3

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 1120 from http://127.0.0.1:41538/je_a/y/collection1_shard1_replica_n5 (previous client) and got 4 from http://127.0.0.1:46233/je_a/y/collection1_shard1_replica_n3
        at __randomizedtesting.SeedInfo.seed([C028F44C042917D7:487CCB96AAD57A2F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1423)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1402)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:157)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
iter3009 21 != 16 expected:<[127.0.0.1:39102_solr, thrasher-T3009_0-0, thrasher-T3009_0-1, thrasher-T3009_0-2, thrasher-T3009_0-3, thrasher-T3009_0-4, thrasher-T3009_1-0, thrasher-T3009_1-1, thrasher-T3009_1-2, thrasher-T3009_1-3, thrasher-T3009_1-4, thrasher-T3009_2-0, thrasher-T3009_2-1, thrasher-T3009_2-2, thrasher-T3009_2-3, thrasher-T3009_2-4, thrasher-T3009_3-0, thrasher-T3009_3-1, thrasher-T3009_3-2, thrasher-T3009_3-3, thrasher-T3009_3-4]> but was:<[127.0.0.1:39102_solr, thrasher-T3009_0-0, thrasher-T3009_0-1, thrasher-T3009_0-2, thrasher-T3009_0-3, thrasher-T3009_0-4, thrasher-T3009_1-0, thrasher-T3009_1-1, thrasher-T3009_1-2, thrasher-T3009_1-3, thrasher-T3009_1-4, thrasher-T3009_2-0, thrasher-T3009_2-1, thrasher-T3009_2-2, thrasher-T3009_2-3, thrasher-T3009_2-4]>

Stack Trace:
java.lang.AssertionError: iter3009 21 != 16 expected:<[127.0.0.1:39102_solr, thrasher-T3009_0-0, thrasher-T3009_0-1, thrasher-T3009_0-2, thrasher-T3009_0-3, thrasher-T3009_0-4, thrasher-T3009_1-0, thrasher-T3009_1-1, thrasher-T3009_1-2, thrasher-T3009_1-3, thrasher-T3009_1-4, thrasher-T3009_2-0, thrasher-T3009_2-1, thrasher-T3009_2-2, thrasher-T3009_2-3, thrasher-T3009_2-4, thrasher-T3009_3-0, thrasher-T3009_3-1, thrasher-T3009_3-2, thrasher-T3009_3-3, thrasher-T3009_3-4]> but was:<[127.0.0.1:39102_solr, thrasher-T3009_0-0, thrasher-T3009_0-1, thrasher-T3009_0-2, thrasher-T3009_0-3, thrasher-T3009_0-4, thrasher-T3009_1-0, thrasher-T3009_1-1, thrasher-T3009_1-2, thrasher-T3009_1-3, thrasher-T3009_1-4, thrasher-T3009_2-0, thrasher-T3009_2-1, thrasher-T3009_2-2, thrasher-T3009_2-3, thrasher-T3009_2-4]>
        at __randomizedtesting.SeedInfo.seed([C028F44C042917D7:D533898D87DEEFAC]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:199)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1659 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexingSequenceNumbers
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexingSequenceNumbers -Dtests.method=testStressConcurrentCommit -Dtests.seed=31D3822807824943 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-JO -Dtests.timezone=Australia/Tasmania -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    606s J2 | TestIndexingSequenceNumbers.testStressConcurrentCommit <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > at __randomizedtesting.SeedInfo.seed([31D3822807824943:4A471897854268F4]:0)
   [junit4]    > at java.util.Arrays.copyOfRange(Arrays.java:3664)
   [junit4]    > at java.lang.String.<init>(String.java:207)
   [junit4]    > at java.lang.StringBuilder.toString(StringBuilder.java:407)
   [junit4]    > at org.apache.lucene.index.CorruptIndexException.<init>(CorruptIndexException.java:62)
   [junit4]    > at org.apache.lucene.index.CorruptIndexException.<init>(CorruptIndexException.java:47)
   [junit4]    > at org.apache.lucene.codecs.CodecUtil.checkFooter(CodecUtil.java:471)
   [junit4]    > at org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat.read(Lucene70SegmentInfoFormat.java:266)
   [junit4]    > at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:361)
   [junit4]    > at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:291)
   [junit4]    > at org.apache.lucene.index.DirectoryReader.listCommits(DirectoryReader.java:277)
   [junit4]    > at org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:233)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexingSequenceNumbers_31D3822807824943-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=MockRandom)}, docValues:{thread=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Direct), ___soft_deletes=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=600, maxMBSortInHeap=5.756445456281477, sim=RandomSimilarity(queryNorm=false): {id=DFR GB1}, locale=ar-JO, timezone=Australia/Tasmania
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=49161512,total=501743616
   [junit4]   2> NOTE: All tests run in this JVM: [TestLucene70SegmentInfoFormat, TestStressIndexing2, TestRegexpRandom, TestExternalCodecs, TestRollingUpdates, TestMultiset, TestIndexWriterForceMerge, TestSwappedIndexFiles, TestDocValues, TestGrowableByteArrayDataOutput, TestRamUsageEstimator, TestStringHelper, TestOmitTf, TestSmallFloat, TestDelegatingAnalyzerWrapper, TestReaderPool, TestIntBlockPool, TestLucene50TermVectorsFormat, TestDuelingCodecsAtNight, TestField, TestIndexWriterDeleteByQuery, TestBagOfPostings, TestLatLonPointDistanceSort, TestAxiomaticSimilarity, TestBlockPostingsFormat3, TestIndexWriterNRTIsCurrent, TestIndexableField, TestPrefixRandom, TestTopFieldCollector, TestTragicIndexWriterDeadlock, TestPriorityQueue, TestDaciukMihovAutomatonBuilder, TestMixedCodecs, TestFieldInfos, TestSpanContainQuery, TestSentinelIntSet, TestSegmentMerger, TestGraphTokenStreamFiniteStrings, TestNRTCachingDirectory, TestByteBlockPool, TestLucene50LiveDocsFormat, TestNearSpansOrdered, TestHighCompressionMode, TestConjunctions, TestPersistentSnapshotDeletionPolicy, TestByteSlices, TestMultiFields, TestTermQuery, TestCodecs, TestRateLimiter, TestSameScoresWithThreads, TestIndexWriterMaxDocs, TestNeverDelete, TestSpanFirstQuery, TestIndexWriterThreadsToSegments, TestSoftDeletesDirectoryReaderWrapper, TestBufferedChecksum, TestBooleanScorer, TestDocIdSetIterator, TestFlex, TestRegExp, TestCustomSearcherSort, TestCustomTermFreq, TestCloseableThreadLocal, TestGeoUtils, TestComplexExplanations, TestTermdocPerf, TestSortedSetSortField, TestTimeLimitingCollector, TestTryDelete, TestAllFilesHaveCodecHeader, TestTermsEnum2, TestRoaringDocIdSet, TestDocCount, Test2BTerms, TestIndexSorting, TestIndexedDISI, TestPackedTokenAttributeImpl, TestUsageTrackingFilterCachingPolicy, TestStopFilter, TestMultiTermConstantScore, TestPerFieldPostingsFormat, TestPerFieldPostingsFormat2, TestBinaryDocument, TestDateTools, TestDocument, TestFloatRange, TestIntRange, TestLatLonDocValuesField, TestLongRange, TestPolygon2D, Test2BBinaryDocValues, Test2BNumericDocValues, Test2BPoints, Test2BPostings, TestExceedMaxTermLength, TestFieldReuse, TestFieldUpdatesBuffer, TestFieldsReader, TestFilterCodecReader, TestFilterDirectoryReader, TestFlushByRamOrCountsPolicy, TestForTooMuchCloning, TestForceMergeForever, TestIndexFileDeleter, TestIndexInput, TestIndexTooManyDocs, TestIndexWriter, TestIndexWriterMergePolicy, TestIndexWriterUnicode, TestIndexingSequenceNumbers]
   [junit4] Completed [390/487 (1!)] on J2 in 697.34s, 8 tests, 1 error <<< FAILURES!

[...truncated 332 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J2-20181212_013035_6722600911664876080666.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid13557.hprof ...
   [junit4] Heap dump file created [572987925 bytes in 2.317 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 12859 lines...]
   [junit4] Suite: org.apache.solr.cloud.RestartWhileUpdatingTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/init-core-data-001
   [junit4]   2> 4092410 WARN  (SUITE-RestartWhileUpdatingTest-seed#[C028F44C042917D7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 4092410 INFO  (SUITE-RestartWhileUpdatingTest-seed#[C028F44C042917D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 4092413 INFO  (SUITE-RestartWhileUpdatingTest-seed#[C028F44C042917D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 4092413 INFO  (SUITE-RestartWhileUpdatingTest-seed#[C028F44C042917D7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4092413 INFO  (SUITE-RestartWhileUpdatingTest-seed#[C028F44C042917D7]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /je_a/y
   [junit4]   2> 4092419 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4092437 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4092438 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4092537 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer start zk server on port:40984
   [junit4]   2> 4092537 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer parse host and port list: localhost:40984
   [junit4]   2> 4092538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer connecting to localhost 40984
   [junit4]   2> 4092589 INFO  (zkConnectionManagerCallback-7656-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4092636 INFO  (zkConnectionManagerCallback-7658-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4092638 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4092639 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 4092640 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4092641 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4092642 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4092643 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4092644 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4092645 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4092646 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4092647 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4092648 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4092649 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4093425 WARN  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 4093441 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 4093442 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4093442 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 4093525 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4093525 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4093526 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4093534 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38bae93{/je_a/y,null,AVAILABLE}
   [junit4]   2> 4093537 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7f74d9d6{HTTP/1.1,[http/1.1]}{127.0.0.1:45600}
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.e.j.s.Server Started @4093624ms
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/tempDir-001/control/data, hostContext=/je_a/y, hostPort=45600, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/control-001/cores}
   [junit4]   2> 4093538 ERROR (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.7.0
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4093538 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-12T05:59:39.238Z
   [junit4]   2> 4093609 INFO  (zkConnectionManagerCallback-7660-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4093611 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4093611 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/control-001/solr.xml
   [junit4]   2> 4093616 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4093616 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4093618 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4096283 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40984/solr
   [junit4]   2> 4096329 INFO  (zkConnectionManagerCallback-7664-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4096373 INFO  (zkConnectionManagerCallback-7666-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4097143 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45600_je_a%2Fy
   [junit4]   2> 4097150 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer Overseer (id=72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) starting
   [junit4]   2> 4097201 INFO  (zkConnectionManagerCallback-7673-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4097204 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4097226 INFO  (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45600_je_a%2Fy
   [junit4]   2> 4097250 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45600_je_a%2Fy
   [junit4]   2> 4097270 INFO  (zkCallback-7672-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4097302 INFO  (zkCallback-7665-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4097323 INFO  (OverseerAutoScalingTriggerThread-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:45600_je_a%2Fy]
   [junit4]   2> 4097393 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4097509 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4097581 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4097618 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4097622 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/control-001/cores
   [junit4]   2> 4097793 INFO  (zkConnectionManagerCallback-7679-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4097802 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4097803 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4097810 INFO  (qtp1933920440-74736) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45600_je_a%252Fy&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4097821 INFO  (OverseerThreadFactory-9897-thread-1-processing-n:127.0.0.1:45600_je_a%2Fy) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4097828 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:45600_je_a%2Fy",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 4097943 INFO  (qtp1933920440-74738) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4097944 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4097945 INFO  (qtp1933920440-74735) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4097946 INFO  (qtp1933920440-74737) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4097954 INFO  (qtp1933920440-74738) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4097970 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4097970 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4097974 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:45600/je_a/y",
   [junit4]   2>   "node_name":"127.0.0.1:45600_je_a%2Fy",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 4099231 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 4099291 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 4109005 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4109098 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 4109098 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4109099 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 4109103 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2ed44744
   [junit4]   2> 4109147 WARN  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4109574 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4109574 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4109595 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
   [junit4]   2> 4109595 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
   [junit4]   2> 4109605 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=5.45703125, floorSegmentMB=0.7109375, forceMergeDeletesPctAllowed=23.758589633940453, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=46.61519360811194
   [junit4]   2> 4109626 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5641a83f[control_collection_shard1_replica_n1] main]
   [junit4]   2> 4109627 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4109628 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4109629 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4109629 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1619624612676501504
   [junit4]   2> 4109633 INFO  (searcherExecutor-9902-thread-1-processing-n:127.0.0.1:45600_je_a%2Fy x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@5641a83f[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4109657 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 4109657 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45600/je_a/y/control_collection_shard1_replica_n1/
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45600/je_a/y/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4109661 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4109662 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72433060057448452-core_node2-n_0000000000
   [junit4]   2> 4109675 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:45600/je_a/y",
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "state":"active"} current state version: 0
   [junit4]   2> 4109681 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45600/je_a/y/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4109683 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4109684 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:45600/je_a/y",
   [junit4]   2>   "node_name":"127.0.0.1:45600_je_a%2Fy",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 4109686 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=11716
   [junit4]   2> 4109692 INFO  (qtp1933920440-74736) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4109785 INFO  (zkCallback-7665-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4109786 INFO  (qtp1933920440-74736) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45600_je_a%252Fy&wt=javabin&version=2} status=0 QTime=11976
   [junit4]   2> 4109824 INFO  (zkCallback-7665-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4109909 INFO  (zkConnectionManagerCallback-7684-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4109911 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4109912 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4109912 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4109913 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4109939 INFO  (OverseerCollectionConfigSetProcessor-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4109940 INFO  (OverseerThreadFactory-9897-thread-2-processing-n:127.0.0.1:45600_je_a%2Fy) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4109963 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"1",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 4110163 WARN  (OverseerThreadFactory-9897-thread-2-processing-n:127.0.0.1:45600_je_a%2Fy) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4110166 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4110167 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=254
   [junit4]   2> 4110177 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
   [junit4]   2> 4111942 INFO  (OverseerCollectionConfigSetProcessor-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4112125 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001 of type NRT
   [junit4]   2> 4112140 WARN  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 4112141 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 4112141 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4112141 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 4112219 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4112220 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4112220 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4112240 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e66ba47{/je_a/y,null,AVAILABLE}
   [junit4]   2> 4112245 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5a45cdf8{HTTP/1.1,[http/1.1]}{127.0.0.1:41705}
   [junit4]   2> 4112245 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.Server Started @4112331ms
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/je_a/y, hostPort=41705, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001/cores}
   [junit4]   2> 4112246 ERROR (closeThreadPool-7685-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.7.0
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4112246 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-12T05:59:57.946Z
   [junit4]   2> 4112277 INFO  (zkConnectionManagerCallback-7687-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4112278 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4112278 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001/solr.xml
   [junit4]   2> 4112283 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4112283 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4112285 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4113491 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40984/solr
   [junit4]   2> 4113517 INFO  (zkConnectionManagerCallback-7691-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4113549 INFO  (zkConnectionManagerCallback-7693-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4113553 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4113557 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.ZkController Publish node=127.0.0.1:41705_je_a%2Fy as DOWN
   [junit4]   2> 4113558 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4113558 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41705_je_a%2Fy
   [junit4]   2> 4113558 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:41705_je_a%2Fy"} current state version: 1
   [junit4]   2> 4113562 INFO  (zkCallback-7683-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4113562 INFO  (zkCallback-7672-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4113562 INFO  (zkCallback-7665-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4113575 INFO  (zkCallback-7692-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4113685 INFO  (zkConnectionManagerCallback-7700-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4113687 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4113688 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4113705 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4113820 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4113900 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4113901 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4113919 INFO  (closeThreadPool-7685-thread-1) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001/cores
   [junit4]   2> 4114005 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-2-001 of type NRT
   [junit4]   2> 4114011 WARN  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 4114011 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 4114011 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4114012 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 4114176 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4114176 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4114177 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4114190 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e6719b6{/je_a/y,null,AVAILABLE}
   [junit4]   2> 4114190 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@59ad5cd4{HTTP/1.1,[http/1.1]}{127.0.0.1:41538}
   [junit4]   2> 4114190 INFO  (closeThreadPool-7685-thread-2) [    ] o.e.j.s.Server Started @4114276ms
   [junit4]   2> 4114190 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/je_a/y, hostPort=41538, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-2-001/cores}
   [junit4]   2> 4114191 ERROR (closeThreadPool-7685-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4114191 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4114191 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.7.0
   [junit4]   2> 4114191 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4114191 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4114191 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-12T05:59:59.891Z
   [junit4]   2> 4114215 INFO  (zkConnectionManagerCallback-7703-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4114226 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4114226 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-2-001/solr.xml
   [junit4]   2> 4114233 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4114233 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4114236 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4115214 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[C028F44C042917D7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-3-001 of type NRT
   [junit4]   2> 4115218 WARN  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 4115230 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 4115230 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 4115230 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 4115286 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4115286 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4115286 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4115298 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e9dbc1a{/je_a/y,null,AVAILABLE}
   [junit4]   2> 4115298 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@308c3ad{HTTP/1.1,[http/1.1]}{127.0.0.1:46233}
   [junit4]   2> 4115298 INFO  (closeThreadPool-7685-thread-1) [    ] o.e.j.s.Server Started @4115384ms
   [junit4]   2> 4115298 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/je_a/y, hostPort=46233, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-3-001/cores}
   [junit4]   2> 4115299 ERROR (closeThreadPool-7685-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4115299 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4115299 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.7.0
   [junit4]   2> 4115299 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4115299 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4115299 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-12T06:00:00.999Z
   [junit4]   2> 4115333 INFO  (zkConnectionManagerCallback-7705-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4115334 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4115334 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-3-001/solr.xml
   [junit4]   2> 4115339 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4115339 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4115341 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4115658 INFO  (closeThreadPool-7685-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40984/solr
   [junit4]   2> 4115693 INFO  (zkConnectionManagerCallback-7709-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4115745 INFO  (zkConnectionManagerCallback-7711-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4115749 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4115753 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.ZkController Publish node=127.0.0.1:41538_je_a%2Fy as DOWN
   [junit4]   2> 4115754 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4115754 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41538_je_a%2Fy
   [junit4]   2> 4115754 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:41538_je_a%2Fy"} current state version: 1
   [junit4]   2> 4115755 INFO  (zkCallback-7692-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4115755 INFO  (zkCallback-7683-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4115755 INFO  (zkCallback-7672-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4115755 INFO  (zkCallback-7665-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4115796 INFO  (zkCallback-7710-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4115802 INFO  (zkCallback-7699-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4116164 INFO  (zkConnectionManagerCallback-7718-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4116165 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 4116166 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4116185 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4116267 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4116340 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4116340 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4116363 INFO  (closeThreadPool-7685-thread-2) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-2-001/cores
   [junit4]   2> 4117970 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40984/solr
   [junit4]   2> 4118001 INFO  (zkConnectionManagerCallback-7723-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4118045 INFO  (zkConnectionManagerCallback-7725-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4118049 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 4118053 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:46233_je_a%2Fy as DOWN
   [junit4]   2> 4118054 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4118054 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46233_je_a%2Fy
   [junit4]   2> 4118055 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:46233_je_a%2Fy"} current state version: 1
   [junit4]   2> 4118056 INFO  (zkCallback-7683-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118056 INFO  (zkCallback-7665-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118056 INFO  (zkCallback-7699-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118056 INFO  (zkCallback-7672-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118056 INFO  (zkCallback-7710-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118056 INFO  (zkCallback-7692-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118071 INFO  (zkCallback-7717-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118085 INFO  (zkCallback-7724-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4118177 INFO  (zkConnectionManagerCallback-7732-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4118179 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 4118181 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40984/solr ready
   [junit4]   2> 4118201 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4118321 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4118408 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4118408 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4118423 INFO  (closeThreadPool-7685-thread-1) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-3-001/cores
   [junit4]   2> 4118994 INFO  (qtp1840883753-74827) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41538_je_a%252Fy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4118994 INFO  (qtp1840883753-74830) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41705_je_a%252Fy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4118994 INFO  (qtp1840883753-74831) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:46233_je_a%252Fy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4119031 INFO  (qtp1840883753-74828) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119032 INFO  (qtp1840883753-74829) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119033 INFO  (qtp1840883753-74828) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119034 INFO  (qtp1840883753-74829) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119034 INFO  (qtp1840883753-74828) [n:127.0.0.1:41538_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119050 INFO  (qtp1880408297-74796) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119051 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119052 INFO  (qtp1880408297-74798) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119052 INFO  (qtp1880408297-74791) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119053 INFO  (qtp1880408297-74795) [n:127.0.0.1:41705_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119060 INFO  (qtp1933920440-74736) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
   [junit4]   2> 4119060 INFO  (qtp1933920440-74735) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 4119061 INFO  (qtp1933920440-74737) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 4119062 INFO  (qtp1933920440-74738) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119063 INFO  (qtp1933920440-74739) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119064 INFO  (qtp1933920440-74736) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119065 INFO  (qtp1933920440-74735) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119066 INFO  (qtp1933920440-74737) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119082 INFO  (qtp1916933601-74840) [n:127.0.0.1:46233_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119083 INFO  (qtp1916933601-74841) [n:127.0.0.1:46233_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119084 INFO  (qtp1916933601-74842) [n:127.0.0.1:46233_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119084 INFO  (qtp1916933601-74843) [n:127.0.0.1:46233_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119085 INFO  (qtp1916933601-74839) [n:127.0.0.1:46233_je_a%2Fy    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 4119085 INFO  (OverseerThreadFactory-9897-thread-3-processing-n:127.0.0.1:45600_je_a%2Fy) [n:127.0.0.1:45600_je_a%2Fy c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:41705_je_a%2Fy for creating new replica of shard shard1 for collection collection1
   [junit4]   2> 4119087 INFO  (OverseerThreadFactory-9897-thread-3-processing-n:127.0.0.1:45600_je_a%2Fy) [n:127.0.0.1:45600_je_a%2Fy c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 4119104 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy    x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4119106 DEBUG (OverseerStateUpdate-72433060057448452-127.0.0.1:45600_je_a%2Fy-n_0000000000) [n:127.0.0.1:45600_je_a%2Fy    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:41705/je_a/y",
   [junit4]   2>   "node_name":"127.0.0.1:41705_je_a%2Fy",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 1
   [junit4]   2> 4120137 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 4120206 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 4120694 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4120737 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 4120737 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@636e1fa0
   [junit4]   2> 4120738 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.RestartWhileUpdatingTest_C028F44C042917D7-001/shard-1-001/cores/collection1_shard1_replica_n1/data/]
   [junit4]   2> 4120758 INFO  (qtp1880408297-74797) [n:127.0.0.1:41705_je_a%2Fy c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.luce

[...truncated too long message...]

/checkout/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/bcprov-jdk15on-1.60.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/byte-buddy-1.9.3.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mockito-core-2.23.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/objenesis-2.6.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.7.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/contrib/analysis-extras/lib/icu4j-62.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_191/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.6.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20181212_045125_5462039913453777120123.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20181212_045125_5461358614503101239380.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1563)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:123)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:997)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:994)
   [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:577: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:267: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/common-build.xml:558: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1572: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1099: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 501 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #391 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 403 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/403/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor550.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #391 to compare, so performing full copy of artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run?
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 1 day 9 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 404 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/404/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [SolrCore, MMapDirectory, MMapDirectory, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [SolrCore, MMapDirectory, MMapDirectory, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)
        at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)
        at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([6B7C49439E196838]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:317)
        at sun.reflect.GeneratedMethodAccessor107.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LIRRollingUpdatesTest.testOldLeaderAndMixedReplicas

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:46029/solr/testMixedReplicas-true

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:46029/solr/testMixedReplicas-true
        at __randomizedtesting.SeedInfo.seed([6B7C49439E196838:CF9891B5E9C62129]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1260)
        at org.apache.solr.cloud.LIRRollingUpdatesTest.realTimeGetDocId(LIRRollingUpdatesTest.java:426)
        at org.apache.solr.cloud.LIRRollingUpdatesTest.assertDocExists(LIRRollingUpdatesTest.java:418)
        at org.apache.solr.cloud.LIRRollingUpdatesTest.assertDocsExistInAllReplicas(LIRRollingUpdatesTest.java:404)
        at org.apache.solr.cloud.LIRRollingUpdatesTest.testLeaderAndMixedReplicas(LIRRollingUpdatesTest.java:321)
        at org.apache.solr.cloud.LIRRollingUpdatesTest.testOldLeaderAndMixedReplicas(LIRRollingUpdatesTest.java:363)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 47 more


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([6B7C49439E196838:E341C03CA4D98995]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState(TestSimTriggerIntegration.java:398)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.core.TestCustomStream.testDynamicLoadingCustomStream

Error Message:
{   "responseHeader":{     "status":500,     "QTime":30395},   "errorMessages":["3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]\n"],   "WARNING":"This response format is experimental.  It is likely to change in the future.",   "error":{     "metadata":[       "error-class","org.apache.solr.common.SolrException",       "root-error-class","org.apache.solr.common.SolrException"],     "msg":"3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]",     "trace":"org.apache.solr.common.SolrException: 3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:768)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:101)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:499)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:345)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:158)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:136)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:502)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)\n\tat java.lang.Thread.run(Thread.java:748)\n",     "code":500}}

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30395},
  "errorMessages":["3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]",
    "trace":"org.apache.solr.common.SolrException: 3 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [http://127.0.0.1:46780/collection1_shard2_replica_n1/, http://127.0.0.1:35010/collection1_shard2_replica_n5/, http://127.0.0.1:41491/collection1_shard1_replica_n2/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:768)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:101)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:499)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:345)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:158)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:136)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:502)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)\n\tat java.lang.Thread.run(Thread.java:748)\n",
    "code":500}}

        at __randomizedtesting.SeedInfo.seed([6B7C49439E196838:1BC9406164E3238F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:180)
        at org.apache.solr.core.TestCustomStream.testDynamicLoadingCustomStream(TestCustomStream.java:58)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
inplace_updatable_float didn't match for replica at client: https://127.0.0.1:42141/fu_rm/collection1 expected:<111.0> but was:<1.0>

Stack Trace:
java.lang.AssertionError: inplace_updatable_float didn't match for replica at client: https://127.0.0.1:42141/fu_rm/collection1 expected:<111.0> but was:<1.0>
        at __randomizedtesting.SeedInfo.seed([6B7C49439E196838:E328769930E505C0]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:1069)
        at org.apache.solr.update.TestInPlaceUpdatesDistrib.outOfOrderUpdatesIndividualReplicaTest(TestInPlaceUpdatesDistrib.java:717)
        at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:165)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14070 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
   [junit4]   2> 3423461 INFO  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[6B7C49439E196838]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/init-core-data-001
   [junit4]   2> 3423461 WARN  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[6B7C49439E196838]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 3423461 INFO  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[6B7C49439E196838]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3423463 INFO  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[6B7C49439E196838]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3423463 INFO  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[6B7C49439E196838]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /vu/iv
   [junit4]   2> 3423464 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ChaosMonkeySafeLeaderWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 6 nodes
   [junit4]   2> 3423534 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3423535 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3423535 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3423643 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer start zk server on port:40088
   [junit4]   2> 3423643 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer parse host and port list: localhost:40088
   [junit4]   2> 3423643 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer connecting to localhost 40088
   [junit4]   2> 3423737 INFO  (zkConnectionManagerCallback-7749-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3423740 INFO  (zkConnectionManagerCallback-7751-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3423774 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 3423776 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 3423809 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3423810 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 3423811 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 3423811 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 3423812 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 3423825 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3423826 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3423827 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 3423827 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 3423866 INFO  (zkConnectionManagerCallback-7755-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3423868 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 3425545 WARN  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3425545 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3425545 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3425545 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3425648 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3425648 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3425648 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3425650 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ee1f8d4{/vu/iv,null,AVAILABLE}
   [junit4]   2> 3425651 WARN  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@57e24858[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@36fc9431{SSL,[ssl, http/1.1]}{127.0.0.1:45716}
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.e.j.s.Server Started @3425774ms
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/tempDir-001/control/data, hostContext=/vu/iv, hostPort=45716, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/control-001/cores}
   [junit4]   2> 3425725 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3425725 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-14T08:22:29.565Z
   [junit4]   2> 3425757 INFO  (zkConnectionManagerCallback-7757-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3425762 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3425762 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/control-001/solr.xml
   [junit4]   2> 3425765 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 3425765 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 3425835 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3430636 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40088/solr
   [junit4]   2> 3430671 INFO  (zkConnectionManagerCallback-7761-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3430797 INFO  (zkConnectionManagerCallback-7763-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3431272 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45716_vu%2Fiv
   [junit4]   2> 3431272 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.Overseer Overseer (id=73834481326948357-127.0.0.1:45716_vu%2Fiv-n_0000000000) starting
   [junit4]   2> 3431453 INFO  (zkConnectionManagerCallback-7770-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3431455 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40088/solr ready
   [junit4]   2> 3431474 INFO  (OverseerStateUpdate-73834481326948357-127.0.0.1:45716_vu%2Fiv-n_0000000000) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45716_vu%2Fiv
   [junit4]   2> 3431510 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45716_vu%2Fiv
   [junit4]   2> 3431623 INFO  (zkCallback-7769-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3431639 INFO  (zkCallback-7762-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3431874 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3431931 INFO  (OverseerAutoScalingTriggerThread-73834481326948357-127.0.0.1:45716_vu%2Fiv-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:45716_vu%2Fiv]
   [junit4]   2> 3432073 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3432147 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3432147 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3432148 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/control-001/cores
   [junit4]   2> 3432531 INFO  (zkConnectionManagerCallback-7776-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3432620 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3432681 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40088/solr ready
   [junit4]   2> 3433136 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45716_vu%252Fiv&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3433192 INFO  (OverseerThreadFactory-6143-thread-1-processing-n:127.0.0.1:45716_vu%2Fiv) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 3433354 INFO  (qtp1907750204-18744) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3433355 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3433356 INFO  (qtp1907750204-18747) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3433382 INFO  (qtp1907750204-18743) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3433383 INFO  (qtp1907750204-18744) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 3433455 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3433455 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 3434748 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 3434879 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 3436012 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 3436174 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 3436174 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3436175 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 3436210 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5216789054391668]
   [junit4]   2> 3436319 WARN  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 3437011 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3437011 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3437013 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3437013 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3437015 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.2314453125, noCFSRatio=0.6168391797921533]
   [junit4]   2> 3437048 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@a9396e2[control_collection_shard1_replica_n1] main]
   [junit4]   2> 3437049 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3437049 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3437050 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3437050 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1619814788250992640
   [junit4]   2> 3437086 INFO  (searcherExecutor-6148-thread-1-processing-n:127.0.0.1:45716_vu%2Fiv x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@a9396e2[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3437106 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 3437106 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 3437141 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3437141 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3437141 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45716/vu/iv/control_collection_shard1_replica_n1/
   [junit4]   2> 3437141 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 3437142 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:45716/vu/iv/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 3437142 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 3437142 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/73834481326948357-core_node2-n_0000000000
   [junit4]   2> 3437144 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45716/vu/iv/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 3437158 INFO  (zkCallback-7762-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3437159 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3437161 INFO  (qtp1907750204-18745) [n:127.0.0.1:45716_vu%2Fiv c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3706
   [junit4]   2> 3437178 INFO  (zkCallback-7762-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3437254 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 3437278 INFO  (OverseerCollectionConfigSetProcessor-73834481326948357-127.0.0.1:45716_vu%2Fiv-n_0000000000) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3437287 INFO  (zkCallback-7762-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3437287 INFO  (zkCallback-7762-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3437339 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45716_vu%252Fiv&wt=javabin&version=2} status=0 QTime=4202
   [junit4]   2> 3437359 INFO  (zkCallback-7762-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 3437550 INFO  (zkConnectionManagerCallback-7781-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3437631 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3437704 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40088/solr ready
   [junit4]   2> 3437704 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 3437870 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3437980 INFO  (OverseerThreadFactory-6143-thread-2-processing-n:127.0.0.1:45716_vu%2Fiv) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 3438364 WARN  (OverseerThreadFactory-6143-thread-2-processing-n:127.0.0.1:45716_vu%2Fiv) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 3438441 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 3438478 INFO  (qtp1907750204-18746) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=607
   [junit4]   2> 3438514 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=2 numOtherReplicas=2
   [junit4]   2> 3439826 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-1-001 of type NRT
   [junit4]   2> 3439861 WARN  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3439861 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3439861 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3439861 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3439977 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3439977 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3439977 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3439977 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2df862f9{/vu/iv,null,AVAILABLE}
   [junit4]   2> 3439978 WARN  (closeThreadPool-7782-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5163ea8c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@69b3d045{SSL,[ssl, http/1.1]}{127.0.0.1:43999}
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.e.j.s.Server Started @3440034ms
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/vu/iv, hostPort=43999, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-1-001/cores}
   [junit4]   2> 3439985 ERROR (closeThreadPool-7782-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 3439985 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3439986 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3439986 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-14T08:22:43.826Z
   [junit4]   2> 3440005 INFO  (OverseerCollectionConfigSetProcessor-73834481326948357-127.0.0.1:45716_vu%2Fiv-n_0000000000) [n:127.0.0.1:45716_vu%2Fiv    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3440013 INFO  (zkConnectionManagerCallback-7784-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3440022 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3440022 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-1-001/solr.xml
   [junit4]   2> 3440042 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 3440042 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 3440043 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3441903 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[6B7C49439E196838]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-2-001 of type NRT
   [junit4]   2> 3441939 WARN  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3441940 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3441940 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3441940 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3442217 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3442217 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3442217 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3442218 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ad69900{/vu/iv,null,AVAILABLE}
   [junit4]   2> 3442218 WARN  (closeThreadPool-7782-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4e583429[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3442255 INFO  (closeThreadPool-7782-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40088/solr
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4e68a592{SSL,[ssl, http/1.1]}{127.0.0.1:32870}
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.e.j.s.Server Started @3442352ms
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/vu/iv, hostPort=32870, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-2-001/cores}
   [junit4]   2> 3442303 ERROR (closeThreadPool-7782-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3442303 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-14T08:22:46.143Z
   [junit4]   2> 3442347 INFO  (zkConnectionManagerCallback-7788-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442361 INFO  (zkConnectionManagerCallback-7790-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442396 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 3442396 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-2-001/solr.xml
   [junit4]   2> 3442400 INFO  (zkConnectionManagerCallback-7792-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442414 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3442469 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 3442469 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 3442470 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3442522 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.ZkController Publish node=127.0.0.1:43999_vu%2Fiv as DOWN
   [junit4]   2> 3442523 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 3442523 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43999_vu%2Fiv
   [junit4]   2> 3442524 INFO  (zkCallback-7762-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442549 INFO  (zkCallback-7769-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442623 INFO  (zkCallback-7780-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442639 INFO  (zkCallback-7791-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3443230 INFO  (zkConnectionManagerCallback-7799-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3443250 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3443251 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at localhost:40088/solr ready
   [junit4]   2> 3443269 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3443579 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3444108 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3444108 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@429f00d
   [junit4]   2> 3444178 INFO  (closeThreadPool-7782-thread-1) [n:127.0.0.1:43999_vu%2Fiv    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_6B7C49439E196838-001/shard-1-001/cores
   [junit4]   2> 3444884 INFO  (closeThreadPool-7782-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=localhost:40088/solr
   [junit4]   2> 3444885 INFO  (zkConnectionManagerCallbac

[...truncated too long message...]

b/apacheds-interceptors-journal-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/bcprov-jdk15on-1.60.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/byte-buddy-1.9.3.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mockito-core-2.23.4.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/objenesis-2.6.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.7.0-SNAPSHOT.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/queryparser/classes/test:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/classes/test:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/contrib/analysis-extras/lib/icu4j-62.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_191/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.6.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20181214_072523_7371230565824643407088.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20181214_072523_7377423159679372667949.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: java.io.IOException: No space left on device
   [junit4] at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
   [junit4] at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
   [junit4] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
   [junit4] at sun.nio.ch.IOUtil.write(IOUtil.java:65)
   [junit4] at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
   [junit4] at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
   [junit4] at java.nio.channels.Channels.writeFully(Channels.java:101)
   [junit4] at java.nio.channels.Channels.access$000(Channels.java:61)
   [junit4] at java.nio.channels.Channels$1.write(Channels.java:174)
   [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
   [junit4] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
   [junit4] at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
   [junit4] at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.Closer.close(Closer.java:214)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1549)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:123)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:997)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:994)
   [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:577: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:267: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/common-build.xml:558: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1572: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1099: At least one slave process threw an exception, first: No space left on device

Total time: 1279 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #391 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 405 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/405/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
        at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:504)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
        at hudson.model.Run.execute(Run.java:1794)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
        at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
        at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
        at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
                at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
                at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
                at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
                at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
                at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
                at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
                at hudson.scm.SCM.checkout(SCM.java:504)
                at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
                at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
                at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
                at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
                at hudson.model.Run.execute(Run.java:1794)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #391 to compare, so performing full copy of artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run?
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 3 days 2 hr old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 406 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/406/

5 tests failed.
FAILED:  org.apache.lucene.document.TestLatLonLineShapeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([FB49E69D266D09FF:7C1E9B12B734757F]:0)
        at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
        at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
        at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
        at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
        at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
        at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:75)
        at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:287)
        at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:131)
        at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
        at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1009)
        at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:709)
        at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:672)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:764)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:797)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:787)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:787)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:787)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:797)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:797)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:787)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:787)
        at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:533)
        at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:898)
        at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
        at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:124)
        at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
        at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
        at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:145)
        at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:142)
        at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:188)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:137)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)


FAILED:  org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testAboveSearchRate

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([D42BDB30CFF267A8:222066F625965D74]:0)
        at org.junit.Assert.fail(Assert.java:86)
        at org.junit.Assert.assertTrue(Assert.java:41)
        at org.junit.Assert.assertTrue(Assert.java:52)
        at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testAboveSearchRate(SearchRateTriggerIntegrationTest.java:273)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Error Message:
ObjectTracker found 8 object(s) that were not released!!! [InternalHttpClient, SolrIndexSearcher, MMapDirectory, TransactionLog, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, MMapDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994)  at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:707)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:625)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)   expected null, but was:<ObjectTracker found 8 object(s) that were not released!!! [InternalHttpClient, SolrIndexSearcher, MMapDirectory, TransactionLog, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, MMapDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994)  at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:707)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:625)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [InternalHttpClient, SolrIndexSearcher, MMapDirectory, TransactionLog, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, MMapDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994)
        at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:707)
        at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:625)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
        at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)
        at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)
        at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)
        at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
        at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
        at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
        at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
        at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
        at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.han

[...truncated too long message...]

server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
   [junit4]    > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
   [junit4]    > at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
   [junit4]    > at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
   [junit4]    > at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
   [junit4]    > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
   [junit4]    > at org.eclipse.jetty.server.Server.handle(Server.java:502)
   [junit4]    > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
   [junit4]    > at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
   [junit4]    > at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
   [junit4]    > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
   [junit4]    > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]    > at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
   [junit4]    > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
   [junit4]    > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
   [junit4]    > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)
   [junit4]    > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)
   [junit4]    > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
   [junit4]    > at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)
   [junit4]    > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)
   [junit4]    > at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
   [junit4]    > at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
   [junit4]    > at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
   [junit4]    > at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
   [junit4]    > at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
   [junit4]    > at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)
   [junit4]    > at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)
   [junit4]    > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
   [junit4]    > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
   [junit4]    > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
   [junit4]    > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
   [junit4]    > at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
   [junit4]    > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
   [junit4]    > at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]    > at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]    > at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
   [junit4]    > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
   [junit4]    > at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
   [junit4]    > at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
   [junit4]    > at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
   [junit4]    > at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
   [junit4]    > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
   [junit4]    > at org.eclipse.jetty.server.Server.handle(Server.java:502)
   [junit4]    > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
   [junit4]    > at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
   [junit4]    > at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
   [junit4]    > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
   [junit4]    > at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
   [junit4]    > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
   [junit4]    > at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
   [junit4]    > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
   [junit4]    > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    > >
   [junit4]    > at __randomizedtesting.SeedInfo.seed([D42BDB30CFF267A8]:0)
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:317)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
   [junit4]    >    1) Thread[id=59136, name=searcherExecutor-4082-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([D42BDB30CFF267A8]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=59136, name=searcherExecutor-4082-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([D42BDB30CFF267A8]:0)
   [junit4] Completed [333/836 (2!)] on J1 in 207.40s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 10741 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid8680.hprof

Total time: 391 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 407 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/407/

5 tests failed.
FAILED:  org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR

Error Message:
Path must not end with / character

Stack Trace:
java.lang.IllegalArgumentException: Path must not end with / character
        at __randomizedtesting.SeedInfo.seed([3672748A295765FD:6CEA4E4C57D7021A]:0)
        at org.apache.zookeeper.common.PathUtils.validatePath(PathUtils.java:58)
        at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1523)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:346)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:71)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:346)
        at org.apache.solr.cloud.LIROnShardRestartTest.testAllReplicasInLIR(LIROnShardRestartTest.java:168)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest

Error Message:
ObjectTracker found 38 object(s) that were not released!!! [TransactionLog, MMapDirectory, MMapDirectory, MMapDirectory, TransactionLog, SolrIndexSearcher, SolrIndexSearcher, TransactionLog, TransactionLog, MMapDirectory, SolrCore, MDCAwareThreadPoolExecutor, MMapDirectory, MMapDirectory, SolrIndexSearcher, MDCAwareThreadPoolExecutor, MMapDirectory, SolrCore, MMapDirectory, MDCAwareThreadPoolExecutor, InternalHttpClient, MMapDirectory, MMapDirectory, SolrIndexSearcher, MMapDirectory, MMapDirectory, SolrIndexSearcher, SolrCore, SolrCore, SolrCore, MDCAwareThreadPoolExecutor, TransactionLog, TransactionLog, MMapDirectory, MDCAwareThreadPoolExecutor, MMapDirectory, TransactionLog, MMapDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:636)  at org.apache.solr.update.DirectUpdateHandler2.delete(DirectUpdateHandler2.java:438)  at org.apache.solr.update.processor.RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:75)  at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:59)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:1007)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1923)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:1458)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:1436)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:124)  at org.apache.solr.update.PeerSync$Updater.applyUpdates(PeerSync.java:640)  at org.apache.solr.update.PeerSync.handleUpdates(PeerSync.java:519)  at org.apache.solr.update.PeerSync.handleResponse(PeerSync.java:375)  at org.apache.solr.update.PeerSync.sync(PeerSync.java:226)  at org.apache.solr.cloud.SyncStrategy.syncWithReplicas(SyncStrategy.java:187)  at org.apache.solr.cloud.SyncStrategy.syncReplicas(SyncStrategy.java:131)  at org.apache.solr.cloud.SyncStrategy.sync(SyncStrategy.java:109)  at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:401)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:171)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:136)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:308)  at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1434)  at org.apache.solr.cloud.ZkController.register(ZkController.java:1176)  at org.apache.solr.cloud.ZkController.register(ZkController.java:1126)  at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:190)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)  at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)  at sun.reflect.GeneratedConstructorAccessor605.newInstance(Unknown Source)  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804)  at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866)  at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)  at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)  at sun.reflect.GeneratedConstructorAccessor605.newInstance(Unknown Source)  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804)  at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866)  at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:896)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:653)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994)  at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:707)  at org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)  at org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:2040)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:2024)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)  at org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureBufferTlog(UpdateLog.java:1311)  at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:629)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1897)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:1458)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:1436)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:124)  at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:200)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:126)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)  at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)  at sun.reflect.GeneratedConstructorAccessor605.newInstance(Unknown Source)  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804)  at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866)  at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)  at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)  at sun.reflect.GeneratedConstructorAccessor605.newInstance(Unknown Source)  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804)  at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866)  at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:690)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1178)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1088)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:734)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:715)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.serve

[...truncated too long message...]

ngs.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null511548766
     [copy] Copying 242 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null511548766
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null511548766

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 82ms :: artifacts dl 16ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 530 minutes 35 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$34.hasMatch(FilePath.java:2678)
        at hudson.FilePath$34.invoke(FilePath.java:2557)
        at hudson.FilePath$34.invoke(FilePath.java:2547)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1036)
                at hudson.FilePath.act(FilePath.java:1025)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1819)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1038)
        at hudson.FilePath.act(FilePath.java:1025)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1819)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 408 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/408/

3 tests failed.
FAILED:  org.apache.solr.DistributedIntervalFacetingTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=3067, name=Thread-2832, state=RUNNABLE, group=TGRP-DistributedIntervalFacetingTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3067, name=Thread-2832, state=RUNNABLE, group=TGRP-DistributedIntervalFacetingTest]
        at __randomizedtesting.SeedInfo.seed([71B28CC4DD28E8D4:F9E6B31E73D4852C]:0)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40826//collection1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://[ff01::114]:33332, http://127.0.0.1:33764//collection1, http://[ff01::083]:33332, http://[ff01::213]:33332, http://127.0.0.1:40826//collection1]
        at __randomizedtesting.SeedInfo.seed([71B28CC4DD28E8D4]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:983)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:998)
        at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:674)


FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
        at __randomizedtesting.SeedInfo.seed([71B28CC4DD28E8D4:F9E6B31E73D4852C]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1014)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1571)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest

Error Message:
ObjectTracker found 2 object(s) that were not released!!! [TransactionLog, TransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)   expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [TransactionLog, TransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)  at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)  at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)  at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)  at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)  at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)  at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)  at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)  at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)  at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)  at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)  at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [TransactionLog, TransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)
        at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)
        at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)
        at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)
        at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)
        at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)
        at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [TransactionLog, TransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)
        at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)
        at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)
        at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552)
        at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:350)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:288)
        at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:235)
        at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:1001)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1222)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:693)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297)
        at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277)
        at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121)
        at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:158)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

>
        at __randomizedtesting.SeedInfo.seed([71B28CC4DD28E8D4]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.failNotNull(Assert.java:755)
        at org.junit.Assert.assertNull(Assert.java:737)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:317)
        at sun.reflect.GeneratedMethodAccessor154.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13449 lines...]
   [junit4] Suite: org.apache.solr.DistributedIntervalFacetingTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/init-core-data-001
   [junit4]   2> 72464 WARN  (SUITE-DistributedIntervalFacetingTest-seed#[71B28CC4DD28E8D4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 72470 INFO  (SUITE-DistributedIntervalFacetingTest-seed#[71B28CC4DD28E8D4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 72472 INFO  (SUITE-DistributedIntervalFacetingTest-seed#[71B28CC4DD28E8D4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-9182 - causes OOM)
   [junit4]   2> 72472 INFO  (SUITE-DistributedIntervalFacetingTest-seed#[71B28CC4DD28E8D4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 72472 INFO  (SUITE-DistributedIntervalFacetingTest-seed#[71B28CC4DD28E8D4]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 73120 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/control/cores/collection1
   [junit4]   2> 73278 WARN  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 73401 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 73402 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 73417 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 73454 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 73454 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 73458 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 73484 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@470ad002{/,null,AVAILABLE}
   [junit4]   2> 73526 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@50746001{HTTP/1.1,[http/1.1]}{127.0.0.1:44440}
   [junit4]   2> 73527 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.Server Started @73768ms
   [junit4]   2> 73527 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {schema=schema-distrib-interval-faceting.xml, solrconfig=solrconfig-basic.xml, hostContext=/, hostPort=44440, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/control/cores}
   [junit4]   2> 73551 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.u.c.SSLCredentialProviderFactory Processing SSL Credential Provider chain: env;sysprop
   [junit4]   2> 73554 ERROR (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 73554 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 73555 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 73555 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 73555 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 73556 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-19T08:05:14.933Z
   [junit4]   2> 73557 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 73560 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/control/solr.xml
   [junit4]   2> 73568 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 73568 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 73587 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 73806 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 73806 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 73849 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 73850 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 73850 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 73855 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/control/cores
   [junit4]   2> 73862 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 73909 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 73953 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 73983 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 74059 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 74060 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from configset /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 74062 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 74063 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/control/cores/collection1/data/]
   [junit4]   2> 74091 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=64.328125, floorSegmentMB=1.412109375, forceMergeDeletesPctAllowed=29.25569642114513, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7238538532428727, deletesPctAllowed=36.95585032395118
   [junit4]   2> 74290 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 74290 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 74292 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7f6b427d
   [junit4]   2> 74311 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71660c2b[collection1] main]
   [junit4]   2> 74312 WARN  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 74313 INFO  (coreLoadExecutor-25-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 74323 INFO  (searcherExecutor-26-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@71660c2b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 74601 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 74615 WARN  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 74616 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 74616 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 74616 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 74622 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 74622 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 74622 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 74624 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5919335e{/,null,AVAILABLE}
   [junit4]   2> 74626 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7d710fc4{HTTP/1.1,[http/1.1]}{127.0.0.1:37652}
   [junit4]   2> 74626 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.e.j.s.Server Started @74868ms
   [junit4]   2> 74626 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {schema=schema-distrib-interval-faceting.xml, solrconfig=solrconfig-basic.xml, hostContext=/, hostPort=37652, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/shard0/cores}
   [junit4]   2> 74627 ERROR (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 74627 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 74627 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 74627 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 74627 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 74627 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-19T08:05:16.004Z
   [junit4]   2> 74628 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 74633 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 74633 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 74636 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 75298 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 75298 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 75411 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 75435 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 75435 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 75460 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/shard0/cores
   [junit4]   2> 75460 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 75466 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 75593 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 75621 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 75670 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 75670 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from configset /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 75671 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cfc26e5
   [junit4]   2> 75671 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.DistributedIntervalFacetingTest_71B28CC4DD28E8D4-001/tempDir-001/shard0/cores/collection1/data/]
   [junit4]   2> 75679 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=64.328125, floorSegmentMB=1.412109375, forceMergeDeletesPctAllowed=29.25569642114513, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7238538532428727, deletesPctAllowed=36.95585032395118
   [junit4]   2> 75822 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 75822 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 75826 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@57e49cfa
   [junit4]   2> 75828 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@f379ddc[collection1] main]
   [junit4]   2> 75840 WARN  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 75841 INFO  (coreLoadExecutor-38-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 75859 INFO  (TEST-DistributedIntervalFacetingTest.test-seed#[71B28CC4DD28E8D4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 75861 INFO  (searcherExecutor-39-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@f379ddc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 76306 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 76322 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 29
   [junit4]   2> 76355 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 76358 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 4
   [junit4]   2> 76365 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 76366 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@45759b79 commitCommandVersion:0
   [junit4]   2> 76369 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1be23fa4[collection1] main]
   [junit4]   2> 76370 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 76376 INFO  (searcherExecutor-26-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1be23fa4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 76376 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 12
   [junit4]   2> 76378 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 76378 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@34b5a825 commitCommandVersion:0
   [junit4]   2> 76381 INFO  (qtp246198087-59) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a79348a[collection1] main]
   [junit4]   2> 76382 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 76386 INFO  (searcherExecutor-39-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1a79348a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 76387 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
   [junit4]   2> 76485 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[0]} 0 95
   [junit4]   2> 76491 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[0]} 0 3
   [junit4]   2> 76496 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[1]} 0 2
   [junit4]   2> 76499 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[1]} 0 1
   [junit4]   2> 76503 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[2]} 0 1
   [junit4]   2> 76506 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[2]} 0 1
   [junit4]   2> 76509 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[3]} 0 1
   [junit4]   2> 76512 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[3]} 0 1
   [junit4]   2> 76516 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[4]} 0 1
   [junit4]   2> 76519 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[4]} 0 1
   [junit4]   2> 76522 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[5]} 0 1
   [junit4]   2> 76531 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[5]} 0 1
   [junit4]   2> 76535 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[6]} 0 1
   [junit4]   2> 76538 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[6]} 0 1
   [junit4]   2> 76541 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[7]} 0 0
   [junit4]   2> 76543 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[7]} 0 1
   [junit4]   2> 76546 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[8]} 0 1
   [junit4]   2> 76549 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[8]} 0 1
   [junit4]   2> 76552 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[9]} 0 1
   [junit4]   2> 76555 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[9]} 0 1
   [junit4]   2> 76570 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[10]} 0 13
   [junit4]   2> 76573 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[10]} 0 1
   [junit4]   2> 76576 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[11]} 0 1
   [junit4]   2> 76586 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[11]} 0 1
   [junit4]   2> 76589 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[12]} 0 1
   [junit4]   2> 76593 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[12]} 0 1
   [junit4]   2> 76598 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[13]} 0 1
   [junit4]   2> 76601 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[13]} 0 1
   [junit4]   2> 76604 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[14]} 0 1
   [junit4]   2> 76607 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[14]} 0 1
   [junit4]   2> 76610 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[15]} 0 1
   [junit4]   2> 76613 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[15]} 0 1
   [junit4]   2> 76616 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[16]} 0 1
   [junit4]   2> 76618 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[16]} 0 1
   [junit4]   2> 76621 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[17]} 0 1
   [junit4]   2> 76623 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[17]} 0 0
   [junit4]   2> 76626 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[18]} 0 1
   [junit4]   2> 76629 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[18]} 0 1
   [junit4]   2> 76631 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[19]} 0 1
   [junit4]   2> 76634 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[19]} 0 1
   [junit4]   2> 76637 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[20]} 0 1
   [junit4]   2> 76639 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[20]} 0 1
   [junit4]   2> 76642 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[21]} 0 1
   [junit4]   2> 76644 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[21]} 0 1
   [junit4]   2> 76652 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[22]} 0 1
   [junit4]   2> 76664 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[22]} 0 1
   [junit4]   2> 76667 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[23]} 0 1
   [junit4]   2> 76672 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[23]} 0 1
   [junit4]   2> 76675 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[24]} 0 0
   [junit4]   2> 76677 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[24]} 0 0
   [junit4]   2> 76695 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[25]} 0 13
   [junit4]   2> 76698 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[25]} 0 1
   [junit4]   2> 76700 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[26]} 0 0
   [junit4]   2> 76703 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[26]} 0 1
   [junit4]   2> 76706 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[27]} 0 1
   [junit4]   2> 76708 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[27]} 0 0
   [junit4]   2> 76711 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[28]} 0 1
   [junit4]   2> 76713 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[28]} 0 0
   [junit4]   2> 76723 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[29]} 0 0
   [junit4]   2> 76725 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[29]} 0 0
   [junit4]   2> 76733 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[30]} 0 1
   [junit4]   2> 76736 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[30]} 0 1
   [junit4]   2> 76738 INFO  (qtp1623263208-40) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[31]} 0 1
   [junit4]   2> 76741 INFO  (qtp246198087-61) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[31]} 0 1
   [junit4]   2> 76743 INFO  (qtp1623263208-36) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[32]} 0 1
   [junit4]   2> 76746 INFO  (qtp246198087-57) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[32]} 0 0
   [junit4]   2> 76748 INFO  (qtp1623263208-37) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[33]} 0 0
   [junit4]   2> 76750 INFO  (qtp246198087-58) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[33]} 0 0
   [junit4]   2> 76753 INFO  (qtp1623263208-38) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[34]} 0 0
   [junit4]   2> 76755 INFO  (qtp246198087-59) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[34]} 0 0
   [junit4]   2> 76757 INFO  (qtp1623263208-39) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[35]} 0 0
   [junit4]   2> 76759 INFO  (qtp246198087-60) [    x:collection1] o.a.s.u.p.LogUpdate

[...truncated too long message...]

ings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null607166688
     [copy] Copying 242 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null607166688
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null607166688

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 65ms :: artifacts dl 3ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 532 minutes 37 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$34.hasMatch(FilePath.java:2678)
        at hudson.FilePath$34.invoke(FilePath.java:2557)
        at hudson.FilePath$34.invoke(FilePath.java:2547)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1036)
                at hudson.FilePath.act(FilePath.java:1025)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1819)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1038)
        at hudson.FilePath.act(FilePath.java:1025)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1819)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 409 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/409/

3 tests failed.
FAILED:  org.apache.lucene.document.TestLatLonLineShapeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([8C26D3F59F1F17AC:B71AE7A0E466B2C]:0)
        at java.util.Arrays.copyOf(Arrays.java:3332)
        at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
        at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
        at java.lang.StringBuilder.append(StringBuilder.java:136)
        at java.lang.StringBuilder.append(StringBuilder.java:131)
        at org.apache.lucene.store.MockIndexInputWrapper.toString(MockIndexInputWrapper.java:224)
        at java.lang.String.valueOf(String.java:2994)
        at java.lang.StringBuilder.append(StringBuilder.java:131)
        at org.apache.lucene.store.BufferedChecksumIndexInput.<init>(BufferedChecksumIndexInput.java:34)
        at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
        at org.apache.lucene.util.bkd.BKDWriter.verifyChecksum(BKDWriter.java:1428)
        at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1884)
        at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1024)
        at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:131)
        at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
        at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:225)
        at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:188)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:137)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2177)
        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2010)
        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1961)
        at org.apache.lucene.document.BaseLatLonShapeTestCase.indexRandomShapes(BaseLatLonShapeTestCase.java:260)
        at org.apache.lucene.document.BaseLatLonShapeTestCase.verify(BaseLatLonShapeTestCase.java:224)
        at org.apache.lucene.document.BaseLatLonShapeTestCase.doTestRandom(BaseLatLonShapeTestCase.java:205)
        at org.apache.lucene.document.BaseLatLonShapeTestCase.testRandomBig(BaseLatLonShapeTestCase.java:181)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
        at __randomizedtesting.SeedInfo.seed([32CB8B231845219C:BA9FB4F9B6B94C64]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1014)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1571)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
        at __randomizedtesting.SeedInfo.seed([32CB8B231845219C:631F72A3A936113B]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.assertTrue(Assert.java:41)
        at org.junit.Assert.assertNotNull(Assert.java:712)
        at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:209)
        at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:168)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10373 lines...]
   [junit4] Suite: org.apache.lucene.document.TestLatLonLineShapeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLatLonLineShapeQueries -Dtests.method=testRandomBig -Dtests.seed=8C26D3F59F1F17AC -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-CO -Dtests.timezone=America/Miquelon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    145s J2 | TestLatLonLineShapeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > at __randomizedtesting.SeedInfo.seed([8C26D3F59F1F17AC:B71AE7A0E466B2C]:0)
   [junit4]    > at java.util.Arrays.copyOf(Arrays.java:3332)
   [junit4]    > at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
   [junit4]    > at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
   [junit4]    > at java.lang.StringBuilder.append(StringBuilder.java:136)
   [junit4]    > at java.lang.StringBuilder.append(StringBuilder.java:131)
   [junit4]    > at org.apache.lucene.store.MockIndexInputWrapper.toString(MockIndexInputWrapper.java:224)
   [junit4]    > at java.lang.String.valueOf(String.java:2994)
   [junit4]    > at java.lang.StringBuilder.append(StringBuilder.java:131)
   [junit4]    > at org.apache.lucene.store.BufferedChecksumIndexInput.<init>(BufferedChecksumIndexInput.java:34)
   [junit4]    > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
   [junit4]    > at org.apache.lucene.util.bkd.BKDWriter.verifyChecksum(BKDWriter.java:1428)
   [junit4]    > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1884)
   [junit4]    > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1024)
   [junit4]    > at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:131)
   [junit4]    > at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
   [junit4]    > at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:225)
   [junit4]    > at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:188)
   [junit4]    > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:137)
   [junit4]    > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
   [junit4]    > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
   [junit4]    > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2177)
   [junit4]    > at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2010)
   [junit4]    > at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1961)
   [junit4]    > at org.apache.lucene.document.BaseLatLonShapeTestCase.indexRandomShapes(BaseLatLonShapeTestCase.java:260)
   [junit4]    > at org.apache.lucene.document.BaseLatLonShapeTestCase.verify(BaseLatLonShapeTestCase.java:224)
   [junit4]    > at org.apache.lucene.document.BaseLatLonShapeTestCase.doTestRandom(BaseLatLonShapeTestCase.java:205)
   [junit4]    > at org.apache.lucene.document.BaseLatLonShapeTestCase.testRandomBig(BaseLatLonShapeTestCase.java:181)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/sandbox/test/J2/temp/lucene.document.TestLatLonLineShapeQueries_8C26D3F59F1F17AC-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=es-CO, timezone=America/Miquelon
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=112000856,total=457179136
   [junit4]   2> NOTE: All tests run in this JVM: [TestTermAutomatonQuery, TestCoveringQuery, TestLatLonLineShapeQueries]
   [junit4] Completed [20/23 (1!)] on J2 in 818.65s, 4 tests, 1 error <<< FAILURES!

[...truncated 4 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/sandbox/test/temp/junit4-J2-20181220_132424_9835720168074007007932.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid28037.hprof ...
   [junit4] Heap dump file created [422704623 bytes in 0.865 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 4034 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/init-core-data-001
   [junit4]   2> 1960936 WARN  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=262 numCloses=262
   [junit4]   2> 1960937 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1960940 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1960941 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1960941 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1961012 WARN  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1961025 WARN  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1961027 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1961081 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_34381_hdfs____.g2ik8b/webapp
   [junit4]   2> 1961923 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34381
   [junit4]   2> 1962191 WARN  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1962194 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1962238 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_39288_datanode____zk8kz/webapp
   [junit4]   2> 1963058 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39288
   [junit4]   2> 1963423 WARN  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1963424 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1963469 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_41851_datanode____ffmu8u/webapp
   [junit4]   2> 1963685 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:38367) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1963763 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x16ed7575a0f33a: from storage DS-0dfcd26b-c974-4acb-a296-3c7a52ef3fba node DatanodeRegistration(127.0.0.1:37855, datanodeUuid=cefdcaac-53bf-4c90-9273-d2a23c1ebb16, infoPort=41210, infoSecurePort=0, ipcPort=37940, storageInfo=lv=-56;cid=testClusterID;nsid=1657376377;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1963763 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x16ed7575a0f33a: from storage DS-82e06e9b-290a-4be5-9839-b14008884d6d node DatanodeRegistration(127.0.0.1:37855, datanodeUuid=cefdcaac-53bf-4c90-9273-d2a23c1ebb16, infoPort=41210, infoSecurePort=0, ipcPort=37940, storageInfo=lv=-56;cid=testClusterID;nsid=1657376377;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1964278 INFO  (SUITE-HdfsRestartWhileUpdatingTest-seed#[32CB8B231845219C]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41851
   [junit4]   2> 1964535 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:38367) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1964551 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x16ed75a49ded76: from storage DS-3f4b0232-a5f5-46b0-a762-c343a3f4492f node DatanodeRegistration(127.0.0.1:39466, datanodeUuid=3811ad6f-21e3-4205-b60b-4fe505760439, infoPort=44912, infoSecurePort=0, ipcPort=41881, storageInfo=lv=-56;cid=testClusterID;nsid=1657376377;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1964551 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x16ed75a49ded76: from storage DS-8a57767f-7558-4708-aac9-cb75bbb3c6ab node DatanodeRegistration(127.0.0.1:39466, datanodeUuid=3811ad6f-21e3-4205-b60b-4fe505760439, infoPort=44912, infoSecurePort=0, ipcPort=41881, storageInfo=lv=-56;cid=testClusterID;nsid=1657376377;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1964891 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1964894 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1964894 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1964995 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer start zk server on port:41001
   [junit4]   2> 1964995 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:41001
   [junit4]   2> 1964995 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 41001
   [junit4]   2> 1965052 INFO  (zkConnectionManagerCallback-7210-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965081 INFO  (zkConnectionManagerCallback-7212-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965090 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1965102 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1965108 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1965112 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1965116 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1965125 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1965127 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1965129 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1965135 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1965140 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1965143 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1965157 INFO  (zkConnectionManagerCallback-7216-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965161 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1965502 WARN  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1965502 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1965502 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1965502 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1965505 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1965505 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1965505 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1965505 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e70f63e{/,null,AVAILABLE}
   [junit4]   2> 1965506 WARN  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1c62eb26[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1965507 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@49bcf790{SSL,[ssl, http/1.1]}{127.0.0.1:34234}
   [junit4]   2> 1965507 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.e.j.s.Server Started @1965628ms
   [junit4]   2> 1965507 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38367/hdfs__localhost_38367__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001_tempDir-002_control_data, hostContext=/, hostPort=34234, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/control-001/cores}
   [junit4]   2> 1965508 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1965508 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1965508 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 1965508 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1965508 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1965508 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-20T15:12:42.984Z
   [junit4]   2> 1965519 INFO  (zkConnectionManagerCallback-7218-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965521 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1965521 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/control-001/solr.xml
   [junit4]   2> 1965527 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1965528 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1965529 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1965709 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41001/solr
   [junit4]   2> 1965711 INFO  (zkConnectionManagerCallback-7222-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965715 INFO  (zkConnectionManagerCallback-7224-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965843 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34234_
   [junit4]   2> 1965844 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.c.Overseer Overseer (id=72480533193621509-127.0.0.1:34234_-n_0000000000) starting
   [junit4]   2> 1965885 INFO  (zkConnectionManagerCallback-7231-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1965896 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1965910 INFO  (OverseerStateUpdate-72480533193621509-127.0.0.1:34234_-n_0000000000) [n:127.0.0.1:34234_    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:34234_
   [junit4]   2> 1965940 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34234_
   [junit4]   2> 1965947 INFO  (zkCallback-7230-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1965967 INFO  (zkCallback-7223-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1965991 INFO  (OverseerAutoScalingTriggerThread-72480533193621509-127.0.0.1:34234_-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:34234_]
   [junit4]   2> 1966133 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1966192 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1966208 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1966208 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1966211 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [n:127.0.0.1:34234_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/control-001/cores
   [junit4]   2> 1966245 INFO  (zkConnectionManagerCallback-7237-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1966247 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1966248 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1966284 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34234_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1966294 INFO  (OverseerThreadFactory-3592-thread-1-processing-n:127.0.0.1:34234_) [n:127.0.0.1:34234_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1966416 INFO  (qtp1372540608-40807) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1966417 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1966419 INFO  (qtp1372540608-40805) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1966420 INFO  (qtp1372540608-40806) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1966421 INFO  (qtp1372540608-40807) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1966427 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1966428 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1969012 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 1969067 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1969261 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1969292 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1969293 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1969293 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1969299 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=4, maxMergedSegmentMB=2.126953125, floorSegmentMB=2.033203125, forceMergeDeletesPctAllowed=8.778281379242841, segmentsPerTier=4.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6433638734610295, deletesPctAllowed=35.09619859525323
   [junit4]   2> 1969328 WARN  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1969413 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1969413 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1969416 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
   [junit4]   2> 1969416 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
   [junit4]   2> 1969421 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=45.73828125, floorSegmentMB=1.0634765625, forceMergeDeletesPctAllowed=3.587483810428255, segmentsPerTier=10.0, maxCFSSegmentSizeMB=1.2509765625, noCFSRatio=1.0, deletesPctAllowed=47.57260644980428
   [junit4]   2> 1969421 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6606a8f8[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1969427 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1969428 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1969433 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1969433 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1620384171330371584
   [junit4]   2> 1969441 INFO  (searcherExecutor-3597-thread-1-processing-n:127.0.0.1:34234_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6606a8f8[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1969448 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1969449 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1969468 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1969468 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1969468 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:34234/control_collection_shard1_replica_n1/
   [junit4]   2> 1969469 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1969469 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:34234/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1969469 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1969470 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72480533193621509-core_node2-n_0000000000
   [junit4]   2> 1969478 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:34234/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1969480 INFO  (zkCallback-7223-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1969480 INFO  (zkCallback-7223-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1969484 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1969487 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3060
   [junit4]   2> 1969496 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1969591 INFO  (zkCallback-7223-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1969591 INFO  (zkCallback-7223-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1969592 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34234_&wt=javabin&version=2} status=0 QTime=3308
   [junit4]   2> 1969592 INFO  (zkCallback-7223-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1969598 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x10180a932230007, likely client has closed socket
   [junit4]   2> 1969604 INFO  (zkConnectionManagerCallback-7242-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1969606 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1969608 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1969608 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1969611 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1969615 INFO  (OverseerThreadFactory-3592-thread-2-processing-n:127.0.0.1:34234_) [n:127.0.0.1:34234_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1969617 INFO  (OverseerCollectionConfigSetProcessor-72480533193621509-127.0.0.1:34234_-n_0000000000) [n:127.0.0.1:34234_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1969826 WARN  (OverseerThreadFactory-3592-thread-2-processing-n:127.0.0.1:34234_) [n:127.0.0.1:34234_    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1969833 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1969843 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=232
   [junit4]   2> 1969853 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
   [junit4]   2> 1970256 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001 of type NRT
   [junit4]   2> 1970260 WARN  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1970260 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1970260 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1970260 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1970262 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1970262 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1970262 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1970263 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@57d0d6ec{/,null,AVAILABLE}
   [junit4]   2> 1970264 WARN  (closeThreadPool-7243-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@66332093[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@458a90bd{SSL,[ssl, http/1.1]}{127.0.0.1:43937}
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.e.j.s.Server Started @1970385ms
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38367/hdfs__localhost_38367__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001_tempDir-002_jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=43937, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001/cores}
   [junit4]   2> 1970265 ERROR (closeThreadPool-7243-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 1970265 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1970266 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1970266 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-20T15:12:47.742Z
   [junit4]   2> 1970273 INFO  (zkConnectionManagerCallback-7245-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1970274 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1970274 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001/solr.xml
   [junit4]   2> 1970279 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1970279 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1970299 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1970497 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-2-001 of type NRT
   [junit4]   2> 1970502 WARN  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1970503 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1970503 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1970503 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1970511 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1970511 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1970511 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1970512 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3588918a{/,null,AVAILABLE}
   [junit4]   2> 1970512 WARN  (closeThreadPool-7243-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7ffcedcb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1970513 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@63443ba1{SSL,[ssl, http/1.1]}{127.0.0.1:39995}
   [junit4]   2> 1970513 INFO  (closeThreadPool-7243-thread-2) [    ] o.e.j.s.Server Started @1970634ms
   [junit4]   2> 1970513 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38367/hdfs__localhost_38367__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=39995, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-2-001/cores}
   [junit4]   2> 1970514 ERROR (closeThreadPool-7243-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1970514 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1970514 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 1970514 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1970514 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1970514 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-20T15:12:47.990Z
   [junit4]   2> 1970527 INFO  (zkConnectionManagerCallback-7247-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1970529 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1970529 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-2-001/solr.xml
   [junit4]   2> 1970534 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1970534 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1970536 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1970645 INFO  (closeThreadPool-7243-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41001/solr
   [junit4]   2> 1970659 INFO  (zkConnectionManagerCallback-7251-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1970673 INFO  (zkConnectionManagerCallback-7253-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1970695 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1970702 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.ZkController Publish node=127.0.0.1:43937_ as DOWN
   [junit4]   2> 1970704 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1970704 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43937_
   [junit4]   2> 1970706 INFO  (zkCallback-7230-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1970713 INFO  (zkCallback-7241-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1970714 INFO  (zkCallback-7223-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1970731 INFO  (zkCallback-7252-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1970930 INFO  (zkConnectionManagerCallback-7260-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1970934 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1970940 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1970956 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1971007 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971024 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971024 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971025 INFO  (TEST-HdfsRestartWhileUpdatingTest.test-seed#[32CB8B231845219C]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-3-001 of type NRT
   [junit4]   2> 1971026 INFO  (closeThreadPool-7243-thread-1) [n:127.0.0.1:43937_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001/cores
   [junit4]   2> 1971029 WARN  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1971029 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1971029 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1971029 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1971031 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1971031 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1971031 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1971032 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cc20cd3{/,null,AVAILABLE}
   [junit4]   2> 1971035 WARN  (closeThreadPool-7243-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@b8391dd[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1971038 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@24ba1b{SSL,[ssl, http/1.1]}{127.0.0.1:38722}
   [junit4]   2> 1971038 INFO  (closeThreadPool-7243-thread-3) [    ] o.e.j.s.Server Started @1971159ms
   [junit4]   2> 1971038 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38367/hdfs__localhost_38367__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001_tempDir-002_jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=38722, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-3-001/cores}
   [junit4]   2> 1971038 ERROR (closeThreadPool-7243-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1971038 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1971039 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 1971039 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1971039 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1971039 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-20T15:12:48.515Z
   [junit4]   2> 1971041 INFO  (zkConnectionManagerCallback-7263-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971042 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1971042 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-3-001/solr.xml
   [junit4]   2> 1971047 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1971047 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1971049 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1971148 INFO  (closeThreadPool-7243-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41001/solr
   [junit4]   2> 1971150 INFO  (zkConnectionManagerCallback-7267-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971152 INFO  (zkConnectionManagerCallback-7269-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971163 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1971170 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.ZkController Publish node=127.0.0.1:38722_ as DOWN
   [junit4]   2> 1971178 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1971178 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38722_
   [junit4]   2> 1971182 INFO  (zkCallback-7252-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971182 INFO  (zkCallback-7241-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971182 INFO  (zkCallback-7223-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971182 INFO  (zkCallback-7230-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971182 INFO  (zkCallback-7268-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971190 INFO  (zkCallback-7259-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1971214 INFO  (zkConnectionManagerCallback-7276-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971215 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1971217 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1971217 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1971221 INFO  (closeThreadPool-7243-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41001/solr
   [junit4]   2> 1971223 INFO  (zkConnectionManagerCallback-7281-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971242 INFO  (zkConnectionManagerCallback-7283-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971250 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971253 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1971258 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.ZkController Publish node=127.0.0.1:39995_ as DOWN
   [junit4]   2> 1971259 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1971259 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39995_
   [junit4]   2> 1971260 INFO  (zkCallback-7230-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971260 INFO  (zkCallback-7241-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971260 INFO  (zkCallback-7252-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971260 INFO  (zkCallback-7223-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971263 INFO  (zkCallback-7259-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971263 INFO  (zkCallback-7268-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971267 INFO  (zkCallback-7275-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971271 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971271 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971274 INFO  (closeThreadPool-7243-thread-3) [n:127.0.0.1:38722_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-3-001/cores
   [junit4]   2> 1971279 INFO  (zkCallback-7282-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1971320 INFO  (zkConnectionManagerCallback-7290-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1971322 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1971325 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41001/solr ready
   [junit4]   2> 1971326 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1971413 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971459 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971459 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1971463 INFO  (closeThreadPool-7243-thread-2) [n:127.0.0.1:39995_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-2-001/cores
   [junit4]   2> 1971626 INFO  (OverseerCollectionConfigSetProcessor-72480533193621509-127.0.0.1:34234_-n_0000000000) [n:127.0.0.1:34234_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1971745 INFO  (qtp1372540608-40805) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43937_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1971750 INFO  (qtp1372540608-40803) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38722_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1971758 INFO  (qtp1372540608-40807) [n:127.0.0.1:34234_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39995_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1971776 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=3
   [junit4]   2> 1971779 INFO  (qtp1372540608-40806) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 1971782 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
   [junit4]   2> 1971789 INFO  (qtp1372540608-40806) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
   [junit4]   2> 1971792 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971796 INFO  (qtp1372540608-40806) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=2
   [junit4]   2> 1971798 INFO  (qtp1372540608-40804) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971800 INFO  (qtp1372540608-40806) [n:127.0.0.1:34234_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
   [junit4]   2> 1971836 INFO  (qtp551482431-40912) [n:127.0.0.1:38722_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971837 INFO  (qtp551482431-40908) [n:127.0.0.1:38722_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971838 INFO  (qtp551482431-40909) [n:127.0.0.1:38722_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971839 INFO  (qtp551482431-40910) [n:127.0.0.1:38722_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971840 INFO  (qtp551482431-40911) [n:127.0.0.1:38722_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971877 INFO  (qtp1408935805-40881) [n:127.0.0.1:39995_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971879 INFO  (qtp1408935805-40880) [n:127.0.0.1:39995_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971881 INFO  (qtp1408935805-40877) [n:127.0.0.1:39995_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971883 INFO  (qtp1408935805-40878) [n:127.0.0.1:39995_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971885 INFO  (qtp1408935805-40879) [n:127.0.0.1:39995_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971909 INFO  (qtp971742266-40865) [n:127.0.0.1:43937_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
   [junit4]   2> 1971920 INFO  (qtp971742266-40868) [n:127.0.0.1:43937_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
   [junit4]   2> 1971923 INFO  (qtp971742266-40861) [n:127.0.0.1:43937_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
   [junit4]   2> 1971926 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971927 INFO  (qtp971742266-40867) [n:127.0.0.1:43937_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1971930 INFO  (OverseerThreadFactory-3592-thread-3-processing-n:127.0.0.1:34234_) [n:127.0.0.1:34234_ c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:43937_ for creating new replica of shard shard1 for collection collection1
   [junit4]   2> 1971940 INFO  (OverseerThreadFactory-3592-thread-3-processing-n:127.0.0.1:34234_) [n:127.0.0.1:34234_ c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 1971952 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_    x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1972992 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 1973031 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 1973265 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1973295 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 1973296 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4da4f37
   [junit4]   2> 1973296 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_32CB8B231845219C-001/shard-1-001/cores/collection1_shard1_replica_n1/data/]
   [junit4]   2> 1973302 INFO  (qtp971742266-40866) [n:127.0.0.1:43937_ c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=4, maxMergedSegmentMB=2.126953125, floorSegmentMB=2.033203125, forceMergeDeletesPctAllowed=8.778281379242841, segmentsPerTier=4.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6433638734610295, deletesPctAllowed=35.09619859525323
   

[...truncated too long message...]

.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1545320003398}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_c(7.7.0):c1:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=7.7.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1545320018524}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_d(7.7.0):c1:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=7.7.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1545320019724}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_e(7.7.0):c5:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=7.7.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1545320026933}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
   [junit4]   2> 3230149 INFO  (commitScheduler-7191-thread-11) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 3230149 INFO  (commitScheduler-7191-thread-11) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3d387f37 commitCommandVersion:0
   [junit4]   2> 3230150 INFO  (commitScheduler-7191-thread-11) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1c3306ad[collection1] realtime]
   [junit4]   2> 3230150 INFO  (commitScheduler-7191-thread-11) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 3235561 INFO  (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[32CB8B231845219C]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testHardCommitWithinAndSoftCommitMaxTimeRapidAdds
   [junit4]   2> 3235562 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 3235562 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1192704207
   [junit4]   2> 3235563 INFO  (coreCloseExecutor-7193-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@45c7a1db
   [junit4]   2> 3235563 INFO  (coreCloseExecutor-7193-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=45c7a1db
   [junit4]   2> 3235563 INFO  (coreCloseExecutor-7193-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2347aee1: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@71439df3
   [junit4]   2> 3235580 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 3235580 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3df9bb7d: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@70b4460e
   [junit4]   2> 3235586 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 3235587 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@353eb39f: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@8c6ffc3
   [junit4]   2> 3235590 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 3235590 INFO  (SUITE-SoftAutoCommitTest-seed#[32CB8B231845219C]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@75a43852: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@57ce14d6
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_32CB8B231845219C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), subject=Lucene50(blocksize=128), intDefault=PostingsFormat(name=Direct), id_i1=PostingsFormat(name=Direct), id=Lucene50(blocksize=128), range_facet_i_dv=Lucene50(blocksize=128), text=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Memory), id_i1=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=942, maxMBSortInHeap=5.829585013790964, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-OM, timezone=Navajo
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=281874736,total=524288000
   [junit4]   2> NOTE: All tests run in this JVM: [RankQueryTest, TestTlogReplica, DocExpirationUpdateProcessorFactoryTest, TermVectorComponentTest, TestBinaryResponseWriter, RestartWhileUpdatingTest, SolrCloudExampleTest, SolrRrdBackendFactoryTest, SyncSliceTest, TestSubQueryTransformerCrossCore, LIRRollingUpdatesTest, TestInfoStreamLogging, TestSQLHandlerNonCloud, TestShortCircuitedRequests, TestBadConfig, ZkFailoverTest, CdcrWithNodesRestartsTest, HdfsBasicDistributedZkTest, TestCollapseQParserPlugin, TestNumericTerms32, TimeZoneUtilsTest, XmlUpdateRequestHandlerTest, TestDynamicLoading, RequestHandlersTest, DistributedQueryComponentOptimizationTest, TestDeleteCollectionOnDownNodes, TestLegacyFieldReuse, TestIBSimilarityFactory, PKIAuthenticationIntegrationTest, CloudMLTQParserTest, AddBlockUpdateTest, TestReversedWildcardFilterFactory, GraphQueryTest, ConfigureRecoveryStrategyTest, ClusterStateTest, AutoscalingHistoryHandlerTest, MoveReplicaHDFSFailoverTest, TestExpandComponent, IndexSizeTriggerTest, DocumentAnalysisRequestHandlerTest, SolrCoreCheckLockOnStartupTest, TestPHPSerializedResponseWriter, TestInPlaceUpdatesDistrib, TestClassicSimilarityFactory, RangeFacetCloudTest, FastVectorHighlighterTest, IndexSchemaTest, DataDrivenBlockJoinTest, MaxSizeAutoCommitTest, IndexBasedSpellCheckerTest, CheckHdfsIndexTest, MetricUtilsTest, DOMUtilTest, TestTrackingShardHandlerFactory, IgnoreLargeDocumentProcessorFactoryTest, TestAnalyzedSuggestions, CdcrReplicationHandlerTest, HdfsRestartWhileUpdatingTest, AddSchemaFieldsUpdateProcessorFactoryTest, DocValuesNotIndexedTest, TestSortableTextField, TestManagedSchemaAPI, TestManagedResourceStorage, SuggestComponentContextFilterQueryTest, TestDocTermOrds, SolrMetricReporterTest, TestNestedUpdateProcessor, DeleteNodeTest, TestSimExecutePlanAction, TestCollationFieldDocValues, TestManagedSynonymGraphFilterFactory, MetricsHistoryIntegrationTest, HLLSerializationTest, TestStandardQParsers, TestQuerySenderListener, TestReplicaProperties, TestPseudoReturnFields, ZkStateWriterTest, TestSchemaSimilarityResource, TestSimComputePlanAction, ProtectedTermFilterFactoryTest, SolrInfoBeanTest, CurrencyRangeFacetCloudTest, DistributedFacetPivotLongTailTest, TestSolrDeletionPolicy1, TestShardHandlerFactory, CreateRoutedAliasTest, TestRTimerTree, ResponseBuilderTest, TestLazyCores, DisMaxRequestHandlerTest, TestImpersonationWithHadoopAuth, TestConfigSetsAPIExclusivity, MetricsConfigTest, TestEmbeddedSolrServerSchemaAPI, ClassificationUpdateProcessorFactoryTest, DocValuesMissingTest, CdcrBidirectionalTest, ReplicaListTransformerTest, CloneFieldUpdateProcessorFactoryTest, TestMacros, StressHdfsTest, TestFieldTypeResource, TestPolicyCloud, TestCloudDeleteByQuery, ExecutePlanActionTest, TestDistribIDF, TestPrepRecovery, TestSlowCompositeReaderWrapper, TestSystemCollAutoCreate, TestCloudJSONFacetSKG, LeaderTragicEventTest, TestSolr4Spatial2, TestFieldCollectionResource, CoreAdminOperationTest, TestDocBasedVersionConstraints, TestCursorMarkWithoutUniqueKey, RecoveryAfterSoftCommitTest, TestCloudPivotFacet, PeerSyncWithIndexFingerprintCachingTest, TestSolrCloudWithDelegationTokens, SuggesterTest, TestCorePropertiesReload, LeaderElectionTest, TestSimpleQParserPlugin, TestUtilizeNode, TolerantUpdateProcessorTest, TestSystemIdResolver, TestRecovery, NoCacheHeaderTest, TestLockTree, SoftAutoCommitTest]
   [junit4] Completed [502/836 (2!)] on J1 in 117.99s, 10 tests, 1 failure <<< FAILURES!

[...truncated 10282 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid28037.hprof

Total time: 307 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 410 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/410/

4 tests failed.
FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
        at __randomizedtesting.SeedInfo.seed([2DC0C97CCCF325AB:A594F6A6620F4853]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1014)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1571)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([2DC0C97CCCF325AB:A6E71AAD8DF58E2F]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:510)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:449)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:417)
        at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=49595, name=qtp1357977583-49595, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=49589, name=qtp1357977583-49589, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=49594, name=qtp1357977583-49594, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=49708, name=Scheduler-1422821673, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=49592, name=qtp1357977583-49592, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=49597, name=Scheduler-2034187847, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=49591, name=qtp1357977583-49591-acceptor-0@199f1002-ServerConnector@249cfe09{HTTP/1.1,[http/1.1]}{127.0.0.1:40240}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=49596, name=qtp1357977583-49596, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=49590, name=qtp1357977583-49590, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=49593, name=qtp1357977583-49593, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
   1) Thread[id=49595, name=qtp1357977583-49595, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=49589, name=qtp1357977583-49589, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=49594, name=qtp1357977583-49594, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=49708, name=Scheduler-1422821673, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=49592, name=qtp1357977583-49592, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=49597, name=Scheduler-2034187847, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=49591, name=qtp1357977583-49591-acceptor-0@199f1002-ServerConnector@249cfe09{HTTP/1.1,[http/1.1]}{127.0.0.1:40240}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=49596, name=qtp1357977583-49596, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=49590, name=qtp1357977583-49590, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=49593, name=qtp1357977583-49593, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([2DC0C97CCCF325AB]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=49597, name=Scheduler-2034187847, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=49589, name=qtp1357977583-49589, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=49590, name=qtp1357977583-49590, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=49708, name=Scheduler-1422821673, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=49597, name=Scheduler-2034187847, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=49589, name=qtp1357977583-49589, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=49590, name=qtp1357977583-49590, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$34/1121776802.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=49708, name=Scheduler-1422821673, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([2DC0C97CCCF325AB]:0)




Build Log:
[...truncated 14752 lines...]
   [junit4] Suite: org.apache.solr.cloud.RestartWhileUpdatingTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/init-core-data-001
   [junit4]   2> 2260618 WARN  (SUITE-RestartWhileUpdatingTest-seed#[2DC0C97CCCF325AB]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 2260618 INFO  (SUITE-RestartWhileUpdatingTest-seed#[2DC0C97CCCF325AB]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2260620 INFO  (SUITE-RestartWhileUpdatingTest-seed#[2DC0C97CCCF325AB]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 2260620 INFO  (SUITE-RestartWhileUpdatingTest-seed#[2DC0C97CCCF325AB]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2260620 INFO  (SUITE-RestartWhileUpdatingTest-seed#[2DC0C97CCCF325AB]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /hta/
   [junit4]   2> 2260624 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2260625 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2260625 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2260725 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer start zk server on port:45367
   [junit4]   2> 2260725 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45367
   [junit4]   2> 2260725 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45367
   [junit4]   2> 2260747 INFO  (zkConnectionManagerCallback-3932-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2260754 INFO  (zkConnectionManagerCallback-3934-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2260756 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2260757 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 2260759 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2260760 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2260761 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2260762 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2260765 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2260766 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2260767 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2260770 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2260772 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2260774 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 2260951 WARN  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2260952 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 2260952 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2260952 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 2260953 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2260953 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2260953 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2260956 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@b049c0d{/hta,null,AVAILABLE}
   [junit4]   2> 2260959 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2449c4ba{HTTP/1.1,[http/1.1]}{127.0.0.1:32881}
   [junit4]   2> 2260959 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.e.j.s.Server Started @2261012ms
   [junit4]   2> 2260959 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/tempDir-001/control/data, replicaType=NRT, hostContext=/hta, hostPort=32881, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/control-001/cores}
   [junit4]   2> 2260959 ERROR (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2260960 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2260960 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 2260960 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2260960 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2260960 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-21T13:15:42.167Z
   [junit4]   2> 2260962 INFO  (zkConnectionManagerCallback-3936-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2260964 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2260964 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/control-001/solr.xml
   [junit4]   2> 2260995 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2260995 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2260997 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2261170 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45367/solr
   [junit4]   2> 2261172 INFO  (zkConnectionManagerCallback-3940-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2261174 INFO  (zkConnectionManagerCallback-3942-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2261263 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32881_hta
   [junit4]   2> 2261263 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.c.Overseer Overseer (id=72485735365869572-127.0.0.1:32881_hta-n_0000000000) starting
   [junit4]   2> 2261271 INFO  (zkConnectionManagerCallback-3949-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2261274 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45367/solr ready
   [junit4]   2> 2261275 INFO  (OverseerStateUpdate-72485735365869572-127.0.0.1:32881_hta-n_0000000000) [n:127.0.0.1:32881_hta    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:32881_hta
   [junit4]   2> 2261279 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32881_hta
   [junit4]   2> 2261281 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2261288 INFO  (zkCallback-3948-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2261309 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2261336 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2261350 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2261351 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2261353 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [n:127.0.0.1:32881_hta    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/control-001/cores
   [junit4]   2> 2261382 INFO  (zkConnectionManagerCallback-3955-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2261384 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2261385 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45367/solr ready
   [junit4]   2> 2261386 INFO  (qtp1594337547-87096) [n:127.0.0.1:32881_hta    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32881_hta&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2261392 INFO  (OverseerThreadFactory-5306-thread-1-processing-n:127.0.0.1:32881_hta) [n:127.0.0.1:32881_hta    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2261498 INFO  (qtp1594337547-87098) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2261499 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2261500 INFO  (qtp1594337547-87095) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2261501 INFO  (qtp1594337547-87097) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2261502 INFO  (qtp1594337547-87098) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2261506 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2261506 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2262529 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.7.0
   [junit4]   2> 2262546 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2262674 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2262700 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2262701 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2262701 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2262706 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6568fde2
   [junit4]   2> 2262714 WARN  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2262826 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2262826 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2262829 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
   [junit4]   2> 2262829 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
   [junit4]   2> 2262832 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=45.470703125, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=12.227870808380043, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=46.044192724720986
   [junit4]   2> 2262832 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@143cd412[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2262834 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2262834 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2262835 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2262835 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1620467404286984192
   [junit4]   2> 2262839 INFO  (searcherExecutor-5311-thread-1-processing-n:127.0.0.1:32881_hta x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@143cd412[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2262843 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 2262843 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 2262849 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2262849 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2262849 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:32881/hta/control_collection_shard1_replica_n1/
   [junit4]   2> 2262849 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2262850 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:32881/hta/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2262850 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2262850 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72485735365869572-core_node2-n_0000000000
   [junit4]   2> 2262852 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:32881/hta/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2262954 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2262955 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2262957 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1451
   [junit4]   2> 2262962 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2262965 INFO  (qtp1594337547-87096) [n:127.0.0.1:32881_hta    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2263064 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2263064 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2263064 INFO  (zkCallback-3941-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2263065 INFO  (qtp1594337547-87096) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32881_hta&wt=javabin&version=2} status=0 QTime=1678
   [junit4]   2> 2263075 INFO  (zkConnectionManagerCallback-3960-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263076 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2263077 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45367/solr ready
   [junit4]   2> 2263077 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2263078 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2263081 INFO  (OverseerThreadFactory-5306-thread-2-processing-n:127.0.0.1:32881_hta) [n:127.0.0.1:32881_hta    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2263083 INFO  (OverseerCollectionConfigSetProcessor-72485735365869572-127.0.0.1:32881_hta-n_0000000000) [n:127.0.0.1:32881_hta    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2263286 WARN  (OverseerThreadFactory-5306-thread-2-processing-n:127.0.0.1:32881_hta) [n:127.0.0.1:32881_hta    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2263288 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 2263289 INFO  (qtp1594337547-87099) [n:127.0.0.1:32881_hta    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=210
   [junit4]   2> 2263290 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
   [junit4]   2> 2263459 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-1-001 of type NRT
   [junit4]   2> 2263460 WARN  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2263460 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 2263460 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2263460 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 2263461 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2263462 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2263462 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2263462 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@40b3d3be{/hta,null,AVAILABLE}
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@24d6a0e3{HTTP/1.1,[http/1.1]}{127.0.0.1:39878}
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.Server Started @2263515ms
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hta, hostPort=39878, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-1-001/cores}
   [junit4]   2> 2263463 ERROR (closeThreadPool-3961-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2263463 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-21T13:15:44.670Z
   [junit4]   2> 2263465 INFO  (zkConnectionManagerCallback-3963-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263466 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2263466 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-1-001/solr.xml
   [junit4]   2> 2263471 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2263471 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2263473 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2263592 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45367/solr
   [junit4]   2> 2263593 INFO  (zkConnectionManagerCallback-3967-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263597 INFO  (zkConnectionManagerCallback-3969-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263602 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2263607 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.ZkController Publish node=127.0.0.1:39878_hta as DOWN
   [junit4]   2> 2263608 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2263608 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39878_hta
   [junit4]   2> 2263609 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2263609 INFO  (zkCallback-3948-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2263609 INFO  (zkCallback-3959-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2263614 INFO  (zkCallback-3968-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2263631 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-2-001 of type NRT
   [junit4]   2> 2263632 WARN  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2263632 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 2263632 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2263633 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 2263634 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2263634 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2263634 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2263635 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6cb8dc60{/hta,null,AVAILABLE}
   [junit4]   2> 2263635 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4bcf0b0{HTTP/1.1,[http/1.1]}{127.0.0.1:43671}
   [junit4]   2> 2263635 INFO  (closeThreadPool-3961-thread-2) [    ] o.e.j.s.Server Started @2263688ms
   [junit4]   2> 2263635 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hta, hostPort=43671, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-2-001/cores}
   [junit4]   2> 2263636 ERROR (closeThreadPool-3961-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2263636 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2263636 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 2263636 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2263636 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2263636 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-21T13:15:44.843Z
   [junit4]   2> 2263638 INFO  (zkConnectionManagerCallback-3973-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263639 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2263639 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-2-001/solr.xml
   [junit4]   2> 2263641 INFO  (zkConnectionManagerCallback-3978-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263642 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2263644 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2263644 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2263645 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45367/solr ready
   [junit4]   2> 2263645 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2263646 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2263675 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2263692 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2263693 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3
   [junit4]   2> 2263695 INFO  (closeThreadPool-3961-thread-1) [n:127.0.0.1:39878_hta    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-1-001/cores
   [junit4]   2> 2263808 INFO  (closeThreadPool-3961-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45367/solr
   [junit4]   2> 2263810 INFO  (zkConnectionManagerCallback-3983-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263817 INFO  (zkConnectionManagerCallback-3985-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263822 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2263827 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.ZkController Publish node=127.0.0.1:43671_hta as DOWN
   [junit4]   2> 2263828 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2263828 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43671_hta
   [junit4]   2> 2263829 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263829 INFO  (zkCallback-3959-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263829 INFO  (zkCallback-3948-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263830 INFO  (zkCallback-3968-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263830 INFO  (zkCallback-3977-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263830 INFO  (zkCallback-3984-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2263850 INFO  (TEST-RestartWhileUpdatingTest.test-seed#[2DC0C97CCCF325AB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-3-001 of type NRT
   [junit4]   2> 2263850 WARN  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 2263850 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 2263850 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 2263850 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 2263852 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2263852 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2263852 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2263852 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@13831fe4{/hta,null,AVAILABLE}
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@65b617a5{HTTP/1.1,[http/1.1]}{127.0.0.1:44592}
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.e.j.s.Server Started @2263905ms
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hta, hostPort=44592, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-3-001/cores}
   [junit4]   2> 2263853 ERROR (closeThreadPool-3961-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.7.0
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2263853 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-12-21T13:15:45.060Z
   [junit4]   2> 2263855 INFO  (zkConnectionManagerCallback-3989-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263856 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2263856 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_2DC0C97CCCF325AB-001/shard-3-001/solr.xml
   [junit4]   2> 2263858 INFO  (zkConnectionManagerCallback-3994-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2263859 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2263860 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45367/solr ready
   [junit4]   2> 2263861 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2263861 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2263861 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2263863 INFO  (closeThreadPool-3961-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@92c76b3, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2263888 INFO  (closeThreadPool-3961-thread-2) [n:127.0.0.1:43671_hta    ] o.a.s.m.r.SolrJmxReporte

[...truncated too long message...]

kout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1010419035
     [copy] Copying 242 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1010419035
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1010419035

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 47ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 284 minutes 32 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$34.hasMatch(FilePath.java:2678)
        at hudson.FilePath$34.invoke(FilePath.java:2557)
        at hudson.FilePath$34.invoke(FilePath.java:2547)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1036)
                at hudson.FilePath.act(FilePath.java:1025)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1819)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1038)
        at hudson.FilePath.act(FilePath.java:1025)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1819)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 411 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/411/

5 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([8E558010A04E9B21:668096F9A8E7A8C]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.assertTrue(Assert.java:41)
        at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeAddedTriggerRestoreState(TestSimTriggerIntegration.java:390)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
        at __randomizedtesting.SeedInfo.seed([8E558010A04E9B21:601BFCA0EB2F6D9]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1014)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1571)
        at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([8E558010A04E9B21:57253C1E14830A5]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:510)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:449)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:417)
        at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=130547, name=qtp849303549-130547-acceptor-0@aa194ce-ServerConnector@7f468ee1{HTTP/1.1,[http/1.1]}{127.0.0.1:34615}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=130546, name=qtp849303549-130546, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$564/324417188.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=130554, name=Scheduler-331653811, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=130549, name=qtp849303549-130549, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=130548, name=qtp849303549-130548, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)     &nbs