[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 33 - Still unstable

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
11 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 33 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/33/

2 tests failed.
FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster

Error Message:
Document mismatch on target after sync expected:<20000> but was:<10100>

Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<20000> but was:<10100>
        at __randomizedtesting.SeedInfo.seed([C754541209085852:13111F4BEE5EEBA9]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster(CdcrBootstrapTest.java:309)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([C754541209085852:4F006BC8A7F435AA]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 9 lines...]
ERROR: Error cloning remote repo 'origin'
hudson.plugins.git.GitException: Command "git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*" returned status code 128:
stdout:
stderr: remote: Counting objects: 66710          
remote: Counting objects: 176416          
remote: Counting objects: 307124          
remote: Counting objects: 410763          
remote: Counting objects: 512560          
remote: Counting objects: 621611          
remote: Counting objects: 754354          
remote: Counting objects: 895731          
remote: Counting objects: 906373, done.        
remote: Compressing objects:   0% (1/183152)          
remote: Compressing objects:   1% (1832/183152)          
remote: Compressing objects:   2% (3664/183152)          
remote: Compressing objects:   3% (5495/183152)          
remote: Compressing objects:   4% (7327/183152)          
remote: Compressing objects:   5% (9158/183152)          
remote: Compressing objects:   6% (10990/183152)          
remote: Compressing objects:   7% (12821/183152)          
remote: Compressing objects:   8% (14653/183152)          
remote: Compressing objects:   9% (16484/183152)          
remote: Compressing objects:  10% (18316/183152)          
remote: Compressing objects:  11% (20147/183152)          
remote: Compressing objects:  12% (21979/183152)          
remote: Compressing objects:  13% (23810/183152)          
remote: Compressing objects:  14% (25642/183152)          
remote: Compressing objects:  15% (27473/183152)          
remote: Compressing objects:  16% (29305/183152)          
remote: Compressing objects:  17% (31136/183152)          
remote: Compressing objects:  18% (32968/183152)          
remote: Compressing objects:  19% (34799/183152)          
remote: Compressing objects:  20% (36631/183152)          
remote: Compressing objects:  21% (38462/183152)          
remote: Compressing objects:  22% (40294/183152)          
remote: Compressing objects:  23% (42125/183152)          
remote: Compressing objects:  24% (43957/183152)          
remote: Compressing objects:  25% (45788/183152)          
remote: Compressing objects:  26% (47620/183152)          
remote: Compressing objects:  27% (49452/183152)          
remote: Compressing objects:  28% (51283/183152)          
remote: Compressing objects:  29% (53115/183152)          
remote: Compressing objects:  30% (54946/183152)          
remote: Compressing objects:  31% (56778/183152)          
remote: Compressing objects:  32% (58609/183152)          
remote: Compressing objects:  33% (60441/183152)          
remote: Compressing objects:  34% (62272/183152)          
remote: Compressing objects:  35% (64104/183152)          
remote: Compressing objects:  36% (65935/183152)          
remote: Compressing objects:  37% (67767/183152)          
remote: Compressing objects:  38% (69598/183152)          
remote: Compressing objects:  39% (71430/183152)          
remote: Compressing objects:  40% (73261/183152)          
remote: Compressing objects:  41% (75093/183152)          
remote: Compressing objects:  42% (76924/183152)          
remote: Compressing objects:  43% (78756/183152)          
remote: Compressing objects:  44% (80587/183152)          
remote: Compressing objects:  45% (82419/183152)          
remote: Compressing objects:  46% (84250/183152)          
remote: Compressing objects:  47% (86082/183152)          
remote: Compressing objects:  48% (87913/183152)          
remote: Compressing objects:  49% (89745/183152)          
remote: Compressing objects:  50% (91576/183152)          
remote: Compressing objects:  51% (93408/183152)          
remote: Compressing objects:  52% (95240/183152)          
remote: Compressing objects:  53% (97071/183152)          
remote: Compressing objects:  54% (98903/183152)          
remote: Compressing objects:  55% (100734/183152)          
remote: Compressing objects:  56% (102566/183152)          
remote: Compressing objects:  57% (104397/183152)          
remote: Compressing objects:  58% (106229/183152)          
remote: Compressing objects:  59% (108060/183152)          
remote: Compressing objects:  60% (109892/183152)          
remote: Compressing objects:  61% (111723/183152)          
remote: Compressing objects:  62% (113555/183152)          
remote: Compressing objects:  63% (115386/183152)          
remote: Compressing objects:  64% (117218/183152)          
remote: Compressing objects:  65% (119049/183152)          
remote: Compressing objects:  66% (120881/183152)          
remote: Compressing objects:  67% (122712/183152)          
remote: Compressing objects:  68% (124544/183152)          
remote: Compressing objects:  69% (126375/183152)          
remote: Compressing objects:  70% (128207/183152)          
remote: Compressing objects:  71% (130038/183152)          
remote: Compressing objects:  72% (131870/183152)          
remote: Compressing objects:  73% (133701/183152)          
remote: Compressing objects:  74% (135533/183152)          
remote: Compressing objects:  75% (137364/183152)          
remote: Compressing objects:  76% (139196/183152)          
remote: Compressing objects:  77% (141028/183152)          
remote: Compressing objects:  78% (142859/183152)          
remote: Compressing objects:  79% (144691/183152)          
remote: Compressing objects:  80% (146522/183152)          
remote: Compressing objects:  81% (148354/183152)          
remote: Compressing objects:  82% (150185/183152)          
remote: Compressing objects:  83% (152017/183152)          
remote: Compressing objects:  84% (153848/183152)          
remote: Compressing objects:  85% (155680/183152)          
remote: Compressing objects:  86% (157511/183152)          
remote: Compressing objects:  86% (158133/183152)          
remote: Compressing objects:  87% (159343/183152)          
remote: Compressing objects:  88% (161174/183152)          
remote: Compressing objects:  88% (162106/183152)          
remote: Compressing objects:  89% (163006/183152)          
remote: Compressing objects:  90% (164837/183152)          
remote: Compressing objects:  91% (166669/183152)          
remote: Compressing objects:  92% (168500/183152)          
remote: Compressing objects:  93% (170332/183152)          
remote: Compressing objects:  94% (172163/183152)          
remote: Compressing objects:  95% (173995/183152)          
remote: Compressing objects:  95% (175737/183152)          
remote: Compressing objects:  96% (175826/183152)          
remote: Compressing objects:  97% (177658/183152)          
remote: Compressing objects:  98% (179489/183152)          
remote: Compressing objects:  98% (181225/183152)          
remote: Compressing objects:  99% (181321/183152)          
remote: Compressing objects: 100% (183152/183152)          
remote: Compressing objects: 100% (183152/183152), done.        
Receiving objects:   0% (1/906373)  
Receiving objects:   1% (9064/906373)  
Receiving objects:   2% (18128/906373), 2.22 MiB | 4.43 MiB/s  
Receiving objects:   3% (27192/906373), 2.22 MiB | 4.43 MiB/s  
Receiving objects:   4% (36255/906373), 2.22 MiB | 4.43 MiB/s  
Receiving objects:   4% (41453/906373), 2.22 MiB | 4.43 MiB/s  
Receiving objects:   5% (45319/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:   6% (54383/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:   7% (63447/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:   8% (72510/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:   9% (81574/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:  10% (90638/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:  11% (99702/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:  12% (108765/906373), 10.19 MiB | 10.17 MiB/s  
Receiving objects:  13% (117829/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  14% (126893/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  15% (135956/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  16% (145020/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  17% (154084/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  18% (163148/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  19% (172211/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  20% (181275/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  21% (190339/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  21% (193137/906373), 16.57 MiB | 11.02 MiB/s  
Receiving objects:  22% (199403/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  23% (208466/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  24% (217530/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  25% (226594/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  26% (235657/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  27% (244721/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  28% (253785/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  29% (262849/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  30% (271912/906373), 21.62 MiB | 10.79 MiB/s  
Receiving objects:  31% (280976/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  32% (290040/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  33% (299104/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  34% (308167/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  35% (317231/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  36% (326295/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  37% (335359/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  38% (344422/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  38% (352840/906373), 26.69 MiB | 10.65 MiB/s  
Receiving objects:  39% (353486/906373), 31.72 MiB | 10.55 MiB/s  
Receiving objects:  40% (362550/906373), 31.72 MiB | 10.55 MiB/s  
Receiving objects:  41% (371613/906373), 31.72 MiB | 10.55 MiB/s  
Receiving objects:  42% (380677/906373), 31.72 MiB | 10.55 MiB/s  
Receiving objects:  42% (384357/906373), 40.72 MiB | 10.11 MiB/s  
fatal: The remote end hung up unexpectedly
fatal: early EOF
fatal: index-pack failed

        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1924)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandWithCredentials(CliGitAPIImpl.java:1643)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.access$300(CliGitAPIImpl.java:71)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$1.execute(CliGitAPIImpl.java:352)
        at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$2.execute(CliGitAPIImpl.java:559)
        at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:153)
        at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:146)
        at hudson.remoting.UserRequest.perform(UserRequest.java:153)
        at hudson.remoting.UserRequest.perform(UserRequest.java:50)
        at hudson.remoting.Request$2.run(Request.java:336)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:68)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        at ......remote call to lucene(Native Method)
        at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1545)
        at hudson.remoting.UserResponse.retrieve(UserRequest.java:253)
        at hudson.remoting.Channel.call(Channel.java:830)
        at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.execute(RemoteGitImpl.java:146)
        at sun.reflect.GeneratedMethodAccessor866.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.invoke(RemoteGitImpl.java:132)
        at com.sun.proxy.$Proxy208.execute(Unknown Source)
        at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1075)
        at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1115)
        at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
        at hudson.scm.SCM.checkout(SCM.java:495)
        at hudson.model.AbstractProject.checkout(AbstractProject.java:1276)
        at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:560)
        at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
        at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:485)
        at hudson.model.Run.execute(Run.java:1735)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:405)
ERROR: Error cloning remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
Fetching upstream changes from git://git.apache.org/lucene-solr.git
 > git --version # timeout=10
 > git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/branch_7_0^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/branch_7_0^{commit} # timeout=10
Checking out Revision ef0f443d4cf002bc24576f1e72556fa06584365f (refs/remotes/origin/branch_7_0)
Commit message: "SOLR-8689: Fix bin/solr.cmd so it can run properly on Java 9"
 > git config core.sparsecheckout # timeout=10
 > git checkout -f ef0f443d4cf002bc24576f1e72556fa06584365f
 > git rev-list 5f5abaf6e1ca956ad5ee8830db4cf7a3bcfde7a8 # timeout=10
Cleaning workspace
 > git rev-parse --verify HEAD # timeout=10
Resetting working tree
 > git reset --hard # timeout=10
 > git clean -fdx # timeout=10
Checking out a fresh workspace because /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data doesn't exist
Cleaning local Directory test-data
Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2017-08-22T05:59:49.471 +0000'
A         README.txt
A         enwiki.random.lines.txt
At revision 1805720

No emails were triggered.
[checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt jenkins-nightly
Buildfile: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml

jenkins-nightly:

-print-java-info:
[java-info] java version "1.8.0_144"
[java-info] Java(TM) SE Runtime Environment (1.8.0_144-b01, Oracle Corporation)
[java-info] Java HotSpot(TM) 64-Bit Server VM (25.144-b01, Oracle Corporation)
[java-info] Test args: []

clean:

clean:

clean:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: Apache Ivy 2.3.0 - 20130110142753 :: http://ant.apache.org/ivy/ ::
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve-groovy:
[ivy:cachepath] :: resolving dependencies :: org.codehaus.groovy#groovy-all-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.codehaus.groovy#groovy-all;2.4.8 in public
[ivy:cachepath] :: resolution report :: resolve 106ms :: artifacts dl 3ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   1   |   0   |   0   |   0   ||   1   |   0   |
        ---------------------------------------------------------------------

-test-with-heapdumps-enabled:
     [echo] Java HotSpot(TM) 64-Bit Server VM: Enabling heap dumps on OutOfMemoryError to dir '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps'.
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve-groovy:

test:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

-clover.load:

resolve-groovy:

-init-totals:

test-core:

-clover.disable:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

-clover.load:

-clover.classpath:

-clover.setup:

clover:

-check-git-state:

-git-cleanroot:

-copy-git-state:
     [copy] Copying 1 file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build

git-autoclean:

resolve:

init:

compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java
    [javac] Compiling 822 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/graph/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/document/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/bkd/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/compressing/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene60/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/index/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/fst/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/tokenattributes/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/search/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene62/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/packed/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/mutable/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene70/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/perfield/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/standard/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/store/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/blocktree/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/search/spans/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/search/similarities/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/util/automaton/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene50/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java/org/apache/lucene/geo/package-info.class
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/java

compile-test-framework:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

init:

compile-lucene-core:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

init:

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

compile-core:

compile-codecs:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

common.init:

compile-lucene-core:

init:

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java
    [javac] Compiling 67 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/bloom/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/memory/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/simpletext/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/blockterms/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/blocktreeords/package-info.class
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/codecs/classes/java

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

common.compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java
    [javac] Compiling 188 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java
    [javac] Note: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java uses or overrides a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/asserting/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/ramonly/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/mockfile/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/cheapbastard/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/mockrandom/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/cranky/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/compressing/dummy/package-info.class
     [copy] Copying 4 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/test-framework/classes/java

compile-core:

compile-test:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/test
    [javac] Compiling 461 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/test
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/classes/test

install-junit4-taskdef:

validate:

resolve-groovy:

-init-totals:

-test:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/temp
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/.caches/test-stats/core
   [junit4] <JUnit4> says ¡Hola! Master seed: B1FCC79D575468D7
   [junit4] Executing 454 suites with 3 JVMs.
   [junit4]
   [junit4] Started J2 PID(1868@localhost).
   [junit4] Started J0 PID(1867@localhost).
   [junit4] Started J1 PID(1869@localhost).
   [junit4] Suite: org.apache.lucene.index.TestFilterDirectoryReader
   [junit4] Completed [1/454] on J1 in 1.67s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestSparseFixedBitDocIdSet
   [junit4] Completed [2/454] on J0 in 4.75s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestExitableDirectoryReader
   [junit4] Completed [3/454] on J0 in 0.01s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestOmitNorms
   [junit4] Completed [4/454] on J0 in 11.48s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestCustomNorms
   [junit4] Completed [5/454] on J1 in 23.27s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDoc
   [junit4] Completed [6/454] on J1 in 1.06s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterMerging
   [junit4] Completed [7/454] on J2 in 37.05s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestDocValuesScoring
   [junit4] Completed [8/454] on J2 in 0.45s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestLongBitSet
   [junit4] Completed [9/454] on J0 in 28.47s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMatchAllDocsQuery
   [junit4] Completed [10/454] on J0 in 0.31s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat
   [junit4] Completed [11/454] on J2 in 56.96s, 27 tests
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:09:16, stalled for 65.8s at: TestNumericDocValuesUpdates.testTonsOfUpdates
   [junit4] Suite: org.apache.lucene.search.TestRegexpQuery
   [junit4] Completed [12/454] on J2 in 0.94s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSortedSetDocValues
   [junit4] Completed [13/454] on J2 in 0.05s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestMultiLevelSkipList
   [junit4] Completed [14/454] on J2 in 0.45s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.document.TestBinaryDocument
   [junit4] Completed [15/454] on J2 in 0.04s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
   [junit4] Completed [16/454] on J2 in 31.05s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterLockRelease
   [junit4] Completed [17/454] on J2 in 0.10s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMinShouldMatch2
   [junit4] Completed [18/454] on J2 in 14.64s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.bkd.TestBKD
   [junit4] Completed [19/454] on J0 in 125.82s, 19 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:11:04, stalled for 61.2s at: TestNRTThreads.testNRTThreads
   [junit4] Suite: org.apache.lucene.index.TestNumericDocValuesUpdates
   [junit4] Completed [20/454] on J1 in 208.40s, 32 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMultiPhraseEnum
   [junit4] Completed [21/454] on J1 in 0.10s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSumDocFreq
   [junit4] Completed [22/454] on J1 in 2.60s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.FuzzyTermOnShortTermsTest
   [junit4] Completed [23/454] on J1 in 0.27s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.TestGraphTokenizers
   [junit4] Completed [24/454] on J1 in 6.62s, 23 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:12:05, stalled for  122s at: TestNRTThreads.testNRTThreads
   [junit4] Suite: org.apache.lucene.analysis.standard.TestStandardAnalyzer
   [junit4] Completed [25/454] on J0 in 106.88s, 36 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestUniqueTermCount
   [junit4] Completed [26/454] on J0 in 0.89s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestNGramPhraseQuery
   [junit4] Completed [27/454] on J0 in 0.06s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanCollection
   [junit4] Completed [28/454] on J0 in 0.28s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.TestCharArrayMap
   [junit4] Completed [29/454] on J0 in 0.23s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTieredMergePolicy
   [junit4] Completed [30/454] on J1 in 37.39s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.Test2BPostingsBytes
   [junit4] IGNOR/A 0.01s J1 | Test2BPostingsBytes.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes ~20GB-30GB of space and 10 minutes))
   [junit4] Completed [31/454] on J1 in 0.02s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestCloseableThreadLocal
   [junit4] Completed [32/454] on J1 in 0.04s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.TestDemo
   [junit4] Completed [33/454] on J1 in 0.08s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanFirstQuery
   [junit4] Completed [34/454] on J1 in 0.11s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestBlendedTermQuery
   [junit4] Completed [35/454] on J1 in 0.18s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSizeBoundedForceMerge
   [junit4] Completed [36/454] on J1 in 0.23s, 11 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMultiThreadTermVectors
   [junit4] Completed [37/454] on J0 in 9.21s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocsWithFieldSet
   [junit4] Completed [38/454] on J0 in 0.14s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSnapshotDeletionPolicy
   [junit4] Completed [39/454] on J0 in 2.53s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestFrequencyTrackingRingBuffer
   [junit4] Completed [40/454] on J0 in 1.56s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.bkd.TestDocIdsWriter
   [junit4] Completed [41/454] on J1 in 17.72s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestParallelLeafReader
   [junit4] Completed [42/454] on J1 in 1.78s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.automaton.TestDeterminizeLexicon
   [junit4] Completed [43/454] on J0 in 15.11s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestNeedsScores
   [junit4] Completed [44/454] on J0 in 0.39s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestMultiDocValues
   [junit4] Completed [45/454] on J1 in 20.44s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestFilterSpans
   [junit4] Completed [46/454] on J1 in 0.02s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestPositionIncrement
   [junit4] Completed [47/454] on J1 in 0.23s, 2 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:13:05, stalled for  182s at: TestNRTThreads.testNRTThreads
   [junit4] Suite: org.apache.lucene.store.TestFileSwitchDirectory
   [junit4] IGNOR/A 0.03s J1 | TestFileSwitchDirectory.testFsyncDoesntCreateNewFiles
   [junit4]    > Assumption #1: test only works for FSDirectory subclasses
   [junit4] IGNOR/A 0.01s J1 | TestFileSwitchDirectory.testPendingDeletions
   [junit4]    > Assumption #1: we can only install VirusCheckingFS on an FSDirectory
   [junit4] Completed [48/454] on J1 in 9.45s, 43 tests, 2 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestReaderClosed
   [junit4] Completed [49/454] on J1 in 0.18s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestNativeFSLockFactory
   [junit4] Completed [50/454] on J0 in 27.63s, 10 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:14:05, stalled for  242s at: TestNRTThreads.testNRTThreads
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:14:17, stalled for 62.9s at: TestDocumentsWriterStallControl.testAccquireReleaseRace
   [junit4] Suite: org.apache.lucene.search.TestBoolean2
   [junit4] Completed [51/454] on J0 in 58.53s, 10 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestNoDeletionPolicy
   [junit4] Completed [52/454] on J0 in 0.14s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestCrashCausesCorruptIndex
   [junit4] Completed [53/454] on J0 in 0.13s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpl
   [junit4] Completed [54/454] on J0 in 0.03s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestManyFields
   [junit4] Completed [55/454] on J0 in 4.61s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestWildcardRandom
   [junit4] Completed [56/454] on J0 in 0.64s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestConstantScoreQuery
   [junit4] Completed [57/454] on J0 in 0.06s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestMergedIterator
   [junit4] Completed [58/454] on J0 in 0.88s, 25 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocumentsWriterStallControl
   [junit4] Completed [59/454] on J1 in 78.83s, 3 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:15:05, stalled for  302s at: TestNRTThreads.testNRTThreads
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:15:41, stalled for 67.7s at: TestIndexSorting.testRandom3
   [junit4] Suite: org.apache.lucene.index.TestNRTThreads
   [junit4] Completed [60/454] on J2 in 352.48s, 1 test
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:16:05, stalled for 67.7s at: TestDoubleRangeFieldQueries.testRandomBig
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterMaxDocs
   [junit4] IGNOR/A 0.01s J2 | TestIndexWriterMaxDocs.testExactlyAtTrueLimit
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes over two hours))
   [junit4] Completed [61/454] on J2 in 15.83s, 21 tests, 1 skipped
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:16:41, stalled for  128s at: TestIndexSorting.testRandom3
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:17:05, stalled for  128s at: TestDoubleRangeFieldQueries.testRandomBig
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50StoredFieldsFormatHighCompression
   [junit4] Completed [62/454] on J2 in 63.78s, 19 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestShardSearching
   [junit4] Completed [63/454] on J2 in 18.26s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestDisjunctionMaxQuery
   [junit4] Completed [64/454] on J2 in 1.11s, 13 tests
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:17:42, stalled for  188s at: TestIndexSorting.testRandom3
   [junit4] Suite: org.apache.lucene.util.TestFixedBitSet
   [junit4] Completed [65/454] on J2 in 31.04s, 19 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestDoubleRangeFieldQueries
   [junit4] Completed [66/454] on J0 in 223.31s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTermVectorsWriter
   [junit4] Completed [67/454] on J0 in 1.60s, 15 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestNumericUtils
   [junit4] Completed [68/454] on J2 in 14.53s, 24 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.automaton.TestCompiledAutomaton
   [junit4] Completed [69/454] on J2 in 1.02s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestNoMergePolicy
   [junit4] Completed [70/454] on J2 in 0.22s, 4 tests
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:18:42, stalled for  248s at: TestIndexSorting.testRandom3
   [junit4] Suite: org.apache.lucene.util.packed.TestPackedInts
   [junit4] Completed [71/454] on J2 in 49.63s, 28 tests, 2 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.TestCharacterUtils
   [junit4] Completed [72/454] on J2 in 0.06s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestBagOfPositions
   [junit4] Completed [73/454] on J0 in 63.74s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMultiPhraseQuery
   [junit4] Completed [74/454] on J0 in 0.64s, 16 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestFastCompressionMode
   [junit4] Completed [75/454] on J2 in 2.46s, 11 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestLongValuesSource
   [junit4] Completed [76/454] on J0 in 3.96s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocsAndPositions
   [junit4] Completed [77/454] on J0 in 2.93s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.Test2BTerms
   [junit4] IGNOR/A 0.03s J0 | Test2BTerms.test2BTerms
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=very slow, use 5g minimum heap))
   [junit4] Completed [78/454] on J0 in 0.04s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestFlex
   [junit4] Completed [79/454] on J0 in 0.53s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestRegexpRandom
   [junit4] Completed [80/454] on J0 in 1.88s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestIntroSelector
   [junit4] Completed [81/454] on J0 in 2.28s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocCount
   [junit4] Completed [82/454] on J0 in 0.98s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestSmallFloat
   [junit4] Completed [83/454] on J0 in 0.32s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestStressIndexing
   [junit4] Completed [84/454] on J0 in 4.71s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocumentWriter
   [junit4] Completed [85/454] on J0 in 1.15s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestUnicodeUtil
   [junit4] Completed [86/454] on J0 in 1.54s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestCollectionUtil
   [junit4] Completed [87/454] on J2 in 41.38s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSimpleSearchEquivalence
   [junit4] Completed [88/454] on J0 in 26.75s, 17 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexableField
   [junit4] Completed [89/454] on J0 in 3.02s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.TestCodecLoadingDeadlock
   [junit4] Completed [90/454] on J0 in 1.24s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4] Completed [91/454] on J0 in 33.32s, 11 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSimpleExplanations
   [junit4] Completed [92/454] on J0 in 2.10s, 70 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestLucene70SegmentInfoFormat
   [junit4] Completed [93/454] on J0 in 3.78s, 16 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexSorting
   [junit4] Completed [94/454] on J1 in 379.43s, 50 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestOmitPositions
   [junit4] Completed [95/454] on J1 in 1.05s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestForTooMuchCloning
   [junit4] Completed [96/454] on J1 in 0.44s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestIndexOrDocValuesQuery
   [junit4] Completed [97/454] on J1 in 0.22s, 1 test
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:21:06, stalled for 71.2s at: TestEarlyTerminatingSortingCollector.testEarlyTermination
   [junit4] Suite: org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs
   [junit4] IGNOR/A 0.01s J1 | TestSimpleExplanationsWithFillerDocs.testMA2
   [junit4]    > Assumption #1: got: <[test is not viable with empty filler docs, null]>, expected: each not null
   [junit4] IGNOR/A 0.00s J1 | TestSimpleExplanationsWithFillerDocs.testMA1
   [junit4]    > Assumption #1: got: <[test is not viable with empty filler docs, null]>, expected: each not null
   [junit4] Completed [98/454] on J1 in 24.56s, 70 tests, 2 skipped
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:21:54, stalled for 70.0s at: TestIndexWriterNRTIsCurrent.testIsCurrentWithThreads
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:22:18, stalled for 60.5s at: TestEarlyTerminatingSortingCollector.testTerminatedEarly
   [junit4] Suite: org.apache.lucene.search.TestEarlyTerminatingSortingCollector
   [junit4] Completed [99/454] on J2 in 157.89s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestTermRangeQuery
   [junit4] Completed [100/454] on J2 in 0.42s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexCommit
   [junit4] Completed [101/454] on J2 in 0.31s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterCommit
   [junit4] Completed [102/454] on J2 in 3.72s, 14 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.TestAssertions
   [junit4] Completed [103/454] on J2 in 0.50s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestCharsRef
   [junit4] Completed [104/454] on J2 in 0.23s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestIndexSearcher
   [junit4] Completed [105/454] on J2 in 0.99s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestParallelCompositeReader
   [junit4] Completed [106/454] on J2 in 1.64s, 11 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:22:54, stalled for  130s at: TestIndexWriterNRTIsCurrent.testIsCurrentWithThreads
   [junit4] Suite: org.apache.lucene.index.TestConsistentFieldNumbers
   [junit4] Completed [107/454] on J2 in 15.15s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestIntArrayDocIdSet
   [junit4] Completed [108/454] on J2 in 0.73s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexManyDocuments
   [junit4] Completed [109/454] on J2 in 2.07s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestIOUtils
   [junit4] Completed [110/454] on J2 in 0.61s, 18 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.bkd.TestMutablePointsReaderUtils
   [junit4] Completed [111/454] on J2 in 4.15s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestSleepingLockWrapper
   [junit4] Completed [112/454] on J2 in 14.48s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestBytesRefHash
   [junit4] Completed [113/454] on J2 in 6.62s, 8 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSegmentTermDocs
   [junit4] Completed [114/454] on J2 in 1.12s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterNRTIsCurrent
   [junit4] Completed [115/454] on J0 in 162.63s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.similarities.TestSimilarityBase
   [junit4] Completed [116/454] on J2 in 2.18s, 31 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestNIOFSDirectory
   [junit4] Completed [117/454] on J2 in 5.57s, 42 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanTermQuery
   [junit4] Completed [118/454] on J2 in 0.04s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.lucene60.TestLucene60PointsFormat
   [junit4] IGNOR/A 0.01s J2 | TestLucene60PointsFormat.testMergeStability
   [junit4]    > Assumption #1: TODO: mess with the parameters and test gets angry!
   [junit4] Completed [119/454] on J2 in 51.74s, 20 tests, 1 skipped
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:24:30, stalled for 63.5s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.util.automaton.TestAutomaton
   [junit4] Completed [120/454] on J2 in 8.86s, 56 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestStringHelper
   [junit4] Completed [121/454] on J2 in 0.05s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSortRescorer
   [junit4] Completed [122/454] on J2 in 3.50s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.similarities.TestSimilarity2
   [junit4] Completed [123/454] on J2 in 0.74s, 8 tests
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:25:07, stalled for 62.4s at: TestPerFieldDocValuesFormat.testRandomAdvanceBinary
   [junit4] Suite: org.apache.lucene.search.TestSearchWithThreads
   [junit4] Completed [124/454] on J2 in 29.53s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMultiset
   [junit4] Completed [125/454] on J2 in 0.04s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestByteBlockPool
   [junit4] Completed [126/454] on J2 in 0.46s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSegmentTermEnum
   [junit4] Completed [127/454] on J2 in 0.09s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestHugeRamFile
   [junit4] Completed [128/454] on J2 in 2.39s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestIntsRef
   [junit4] Completed [129/454] on J2 in 0.02s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.Test2BPositions
   [junit4] IGNOR/A 0.00s J2 | Test2BPositions.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=uses lots of space and takes a few minutes))
   [junit4] Completed [130/454] on J2 in 0.00s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestDateSort
   [junit4] Completed [131/454] on J2 in 0.27s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.geo.TestGeoEncodingUtils
   [junit4] Completed [132/454] on J2 in 1.99s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestSetOnce
   [junit4] Completed [133/454] on J2 in 0.08s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestFilterWeight
   [junit4] Completed [134/454] on J2 in 0.02s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSortedSetSortField
   [junit4] Completed [135/454] on J2 in 0.12s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.Test2BPagedBytes
   [junit4] IGNOR/A 0.00s J2 | Test2BPagedBytes.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=You must increase heap to > 2 G to run this))
   [junit4] Completed [136/454] on J2 in 0.00s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestMultiTermsEnum
   [junit4] Completed [137/454] on J2 in 0.03s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestAllFilesHaveCodecHeader
   [junit4] Completed [138/454] on J2 in 1.50s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestReadOnlyIndex
   [junit4] Completed [139/454] on J2 in 0.33s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.document.TestField
   [junit4] Completed [140/454] on J2 in 0.11s, 23 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestSloppyMath
   [junit4] Completed [141/454] on J2 in 4.48s, 8 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:25:31, stalled for  124s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:26:31, stalled for  184s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:26:43, stalled for 61.7s at: TestLongRangeFieldQueries.testRandomBig
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:27:31, stalled for  244s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:27:43, stalled for  122s at: TestLongRangeFieldQueries.testRandomBig
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat
   [junit4] Completed [142/454] on J1 in 393.78s, 110 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestFilterIterator
   [junit4] Completed [143/454] on J1 in 0.12s, 8 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestElevationComparator
   [junit4] Completed [144/454] on J1 in 0.09s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.automaton.LimitedFiniteStringsIteratorTest
   [junit4] Completed [145/454] on J1 in 0.40s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestAddIndexes
   [junit4] Completed [146/454] on J1 in 9.17s, 25 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTermdocPerf
   [junit4] Completed [147/454] on J1 in 0.04s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches
   [junit4] Completed [148/454] on J1 in 0.16s, 30 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestLongRangeFieldQueries
   [junit4] Completed [149/454] on J2 in 183.72s, 5 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:28:31, stalled for  304s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterDelete
   [junit4] Completed [150/454] on J1 in 30.85s, 25 tests, 1 skipped
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:29:31, stalled for 65.3s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:29:31, stalled for  364s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:29:55, stalled for 66.7s at: TestMixedDocValuesUpdates.testTonsOfUpdates
   [junit4] Suite: org.apache.lucene.index.TestMixedDocValuesUpdates
   [junit4] Completed [151/454] on J1 in 101.62s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestCustomSearcherSort
   [junit4] Completed [152/454] on J1 in 13.00s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestQueryBuilder
   [junit4] Completed [153/454] on J1 in 0.09s, 23 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestPositiveScoresOnlyCollector
   [junit4] Completed [154/454] on J1 in 0.03s, 1 test
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:30:31, stalled for  424s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:30:31, stalled for  125s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] Suite: org.apache.lucene.index.TestPointValues
   [junit4] Completed [155/454] on J1 in 12.78s, 32 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestPrefixInBooleanQuery
   [junit4] Completed [156/454] on J1 in 4.07s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSortedSetSelector
   [junit4] Completed [157/454] on J1 in 0.30s, 15 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestRollback
   [junit4] Completed [158/454] on J1 in 0.05s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterFromReader
   [junit4] Completed [159/454] on J1 in 2.09s, 12 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestMultiTermConstantScore
   [junit4] Completed [160/454] on J1 in 7.40s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.automaton.TestMinimize
   [junit4] Completed [161/454] on J1 in 8.23s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.TestReusableStringReader
   [junit4] Completed [162/454] on J1 in 0.05s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.document.TestFieldType
   [junit4] Completed [163/454] on J1 in 0.03s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSwappedIndexFiles
   [junit4] Completed [164/454] on J1 in 0.23s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSegmentInfos
   [junit4] Completed [165/454] on J1 in 0.06s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestTimeLimitingCollector
   [junit4] Completed [166/454] on J1 in 10.67s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestStressNRT
   [junit4] Completed [167/454] on J1 in 5.39s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.TestExternalCodecs
   [junit4] Completed [168/454] on J1 in 2.61s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestTermQuery
   [junit4] Completed [169/454] on J1 in 0.05s, 2 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:31:31, stalled for  185s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:31:31, stalled for  484s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.TestBooleanOr
   [junit4] Completed [170/454] on J1 in 20.61s, 7 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:32:31, stalled for  544s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:32:31, stalled for  246s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:32:43, stalled for 60.3s at: Test2BPostings.test
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:33:31, stalled for  306s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:33:31, stalled for  604s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:33:43, stalled for  120s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:34:31, stalled for  664s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:34:31, stalled for  366s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:34:43, stalled for  181s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:35:32, stalled for  725s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:35:32, stalled for  426s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:35:44, stalled for  241s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:36:32, stalled for  785s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:36:32, stalled for  486s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:36:44, stalled for  301s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:37:32, stalled for  845s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:37:32, stalled for  546s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:37:44, stalled for  361s at: Test2BPostings.test
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:38:32, stalled for  606s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:38:32, stalled for  905s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:38:44, stalled for  421s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:39:32, stalled for  965s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:39:32, stalled for  666s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:39:44, stalled for  481s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:40:32, stalled for 1025s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:40:32, stalled for  726s at: TestIndexWriterReader.testDuringAddDelete
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:40:44, stalled for  541s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:41:32, stalled for 1085s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:41:44, stalled for  601s at: Test2BPostings.test
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterReader
   [junit4] Completed [171/454] on J2 in 815.59s, 23 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestExceedMaxTermLength
   [junit4] Completed [172/454] on J2 in 0.18s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanNearQuery
   [junit4] Completed [173/454] on J2 in 0.05s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestBasics
   [junit4] Completed [174/454] on J2 in 6.47s, 26 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:42:32, stalled for 1145s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:42:44, stalled for  661s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:43:32, stalled for 1205s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:43:44, stalled for  721s at: Test2BPostings.test
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:43:44, stalled for 66.7s at: TestLucene70NormsFormat.testSparseNCommonBig
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:44:32, stalled for 1266s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:44:45, stalled for  127s at: TestLucene70NormsFormat.testSparseNCommonBig
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:44:45, stalled for  782s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:45:33, stalled for 1326s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:45:45, stalled for  187s at: TestLucene70NormsFormat.testSparseNCommonBig
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:45:45, stalled for  842s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:46:33, stalled for 1386s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:46:45, stalled for  902s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:47:33, stalled for 1446s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:47:45, stalled for  962s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:48:33, stalled for 1506s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:48:45, stalled for 1022s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:49:33, stalled for 1566s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:49:45, stalled for 1082s at: Test2BPostings.test
   [junit4] Suite: org.apache.lucene.index.Test2BPostings
   [junit4] Completed [175/454] on J1 in 1096.41s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestIntroSorter
   [junit4] Completed [176/454] on J1 in 0.38s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.TestMergeSchedulerExternal
   [junit4] Completed [177/454] on J1 in 0.58s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestRateLimiter
   [junit4] IGNOR/A 0.83s J1 | TestRateLimiter.testThreads
   [junit4]    > Assumption #1: actualMBPerSec=16.406308037857507 targetMBPerSec=20.92510374737457
   [junit4] Completed [178/454] on J1 in 0.87s, 2 tests, 1 skipped
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:50:33, stalled for 1626s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:51:09, stalled for 68.0s at: TestSearcherManager.testSearcherManager
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:51:33, stalled for 1686s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:52:10, stalled for 69.5s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:52:10, stalled for  128s at: TestSearcherManager.testSearcherManager
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:52:34, stalled for 1747s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:53:10, stalled for  130s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:53:10, stalled for  189s at: TestSearcherManager.testSearcherManager
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:53:34, stalled for 1807s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:54:10, stalled for  249s at: TestSearcherManager.testSearcherManager
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:54:10, stalled for  190s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:54:34, stalled for 1867s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:55:10, stalled for  250s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:55:10, stalled for  309s at: TestSearcherManager.testSearcherManager
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:55:34, stalled for 1927s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:56:11, stalled for  310s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] Suite: org.apache.lucene.search.TestSearcherManager
   [junit4] Completed [179/454] on J1 in 382.02s, 11 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.mutable.TestMutableValues
   [junit4] Completed [180/454] on J1 in 0.04s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestPriorityQueue
   [junit4] Completed [181/454] on J1 in 0.78s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestCheckIndex
   [junit4] Completed [182/454] on J1 in 1.02s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestBooleanQuery
   [junit4] Completed [183/454] on J1 in 0.75s, 22 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestInPlaceMergeSorter
   [junit4] Completed [184/454] on J1 in 0.46s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanOrQuery
   [junit4] Completed [185/454] on J1 in 0.04s, 3 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:56:35, stalled for 1988s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.TestFieldValueQuery
   [junit4] Completed [186/454] on J1 in 27.35s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestBytesRefAttImpl
   [junit4] Completed [187/454] on J1 in 0.03s, 1 test
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T06:57:11, stalled for  371s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:57:35, stalled for 2048s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4] Completed [188/454] on J0 in 2065.76s, 3 tests
   [junit4]
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:57:59, stalled for 62.9s at: TestFSTs.testBigSet
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:58:59, stalled for 66.3s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J1 PID(1869@localhost): 2017-08-22T06:58:59, stalled for  123s at: TestFSTs.testBigSet
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestLucene70NormsFormat
   [junit4] IGNOR/A 0.01s J2 | TestLucene70NormsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] Completed [189/454] on J2 in 1017.83s, 29 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestPayloads
   [junit4] Completed [190/454] on J2 in 0.24s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestLSBRadixSorter
   [junit4] Completed [191/454] on J2 in 0.55s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestBagOfPostings
   [junit4] Completed [192/454] on J2 in 9.97s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestCharsRefBuilder
   [junit4] Completed [193/454] on J2 in 0.02s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocValues
   [junit4] Completed [194/454] on J2 in 0.15s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestAttributeSource
   [junit4] Completed [195/454] on J2 in 0.04s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriter
   [junit4] Completed [196/454] on J2 in 13.53s, 72 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.util.fst.TestBytesStore
   [junit4] Completed [197/454] on J2 in 29.23s, 1 test
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T06:59:59, stalled for  127s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterWithThreads
   [junit4] IGNOR/A 0.04s J2 | TestIndexWriterWithThreads.testOpenTwoIndexWritersOnDifferentThreads
   [junit4]    > Assumption #1: aborting test: timeout obtaining lock
   [junit4] Completed [198/454] on J2 in 12.21s, 12 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.util.fst.TestFSTs
   [junit4] Completed [199/454] on J1 in 219.18s, 20 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanExplanations
   [junit4] Completed [200/454] on J1 in 1.95s, 30 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTermVectorsReader
   [junit4] Completed [201/454] on J1 in 0.71s, 13 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestNorms
   [junit4] Completed [202/454] on J1 in 8.84s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestAutomatonQueryUnicode
   [junit4] Completed [203/454] on J1 in 0.04s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.util.TestSparseFixedBitSet
   [junit4] Completed [204/454] on J1 in 3.90s, 11 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:00:59, stalled for  187s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] Suite: org.apache.lucene.store.TestMultiMMap
   [junit4] Completed [205/454] on J1 in 21.91s, 54 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestFlushByRamOrCountsPolicy
   [junit4] Completed [206/454] on J1 in 8.33s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSloppyPhraseQuery
   [junit4] Completed [207/454] on J1 in 2.70s, 9 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterMergePolicy
   [junit4] Completed [208/454] on J1 in 7.04s, 7 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:01:59, stalled for  247s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:01:59, stalled for 63.1s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecs
   [junit4] Completed [209/454] on J1 in 69.83s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestPersistentSnapshotDeletionPolicy
   [junit4] Completed [210/454] on J1 in 2.02s, 13 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestAtomicUpdate
   [junit4] Completed [211/454] on J1 in 5.85s, 1 test
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:02:59, stalled for  307s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:02:59, stalled for  123s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.index.TestStressAdvance
   [junit4] Completed [212/454] on J1 in 22.59s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDirectoryReaderReopen
   [junit4] Completed [213/454] on J1 in 3.56s, 14 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestSearchAfter
   [junit4] Completed [214/454] on J1 in 2.48s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestControlledRealTimeReopenThread
   [junit4] IGNOR/A 0.00s J0 | TestControlledRealTimeReopenThread.testCRTReopen
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/LUCENE-5737))
   [junit4] Completed [215/454] on J0 in 326.84s, 6 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTerms
   [junit4] Completed [216/454] on J0 in 0.39s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestConcurrentMergeScheduler
   [junit4] Completed [217/454] on J1 in 20.54s, 16 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestHighCompressionMode
   [junit4] Completed [218/454] on J1 in 2.54s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestNeverDelete
   [junit4] Completed [219/454] on J1 in 5.93s, 1 test
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:04:00, stalled for  183s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.search.TestTopDocsMerge
   [junit4] Completed [220/454] on J1 in 42.45s, 4 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:04:36, stalled for 65.4s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.index.TestLongPostings
   [junit4] Completed [221/454] on J1 in 19.44s, 2 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:05:00, stalled for  243s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:05:36, stalled for  126s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.index.TestRollingUpdates
   [junit4] Completed [222/454] on J1 in 48.97s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterUnicode
   [junit4] Completed [223/454] on J1 in 8.86s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestNRTReaderWithThreads
   [junit4] Completed [224/454] on J1 in 1.99s, 1 test
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:06:00, stalled for  304s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat
   [junit4] IGNOR/A 0.00s J1 | TestPerFieldPostingsFormat.testPostingsEnumReuse
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] IGNOR/A 0.01s J1 | TestPerFieldPostingsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] Completed [225/454] on J1 in 44.19s, 25 tests, 2 skipped
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:06:36, stalled for  186s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.util.TestArrayUtil
   [junit4] Completed [226/454] on J1 in 8.76s, 14 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.util.automaton.TestLevenshteinAutomata
   [junit4] Completed [227/454] on J1 in 2.51s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestByteSlices
   [junit4] Completed [228/454] on J1 in 8.34s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestBufferedIndexInput
   [junit4] Completed [229/454] on J1 in 0.53s, 4 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:07:00, stalled for  364s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:07:36, stalled for  246s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.util.TestWeakIdentityMap
   [junit4] Completed [230/454] on J1 in 46.12s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDirectoryReader
   [junit4] Completed [231/454] on J1 in 1.89s, 24 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.store.TestLockFactory
   [junit4] Completed [232/454] on J1 in 0.04s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSegmentMerger
   [junit4] Completed [233/454] on J1 in 1.00s, 3 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestPhraseQuery
   [junit4] Completed [234/454] on J1 in 2.55s, 21 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestComplexExplanations
   [junit4] Completed [235/454] on J1 in 1.38s, 21 tests
   [junit4]
   [junit4] HEARTBEAT J2 PID(1868@localhost): 2017-08-22T07:08:00, stalled for  424s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.search.TestRegexpRandom2
   [junit4] Completed [236/454] on J1 in 16.11s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestSegmentReader
   [junit4] Completed [237/454] on J1 in 0.78s, 7 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestFastDecompressionMode
   [junit4] Completed [238/454] on J1 in 3.46s, 11 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.search.TestFieldCacheRewriteMethod
   [junit4] Completed [239/454] on J1 in 12.59s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTransactions
   [junit4] Completed [240/454] on J1 in 2.66s, 1 test
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestCrash
   [junit4] Completed [241/454] on J1 in 1.20s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat2
   [junit4] Completed [242/454] on J1 in 5.18s, 6 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.analysis.TestToken
   [junit4] Completed [243/454] on J1 in 0.02s, 5 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestFieldsReader
   [junit4] Completed [244/454] on J1 in 0.55s, 2 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestDocValuesIndexing
   [junit4] Completed [245/454] on J1 in 1.03s, 34 tests
   [junit4]
   [junit4] HEARTBEAT J0 PID(1867@localhost): 2017-08-22T07:08:36, stalled for  306s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.search.TestBooleanMinShouldMatch
   [junit4] Completed [246/454] on J1 in 5.02s, 17 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestTermsEnum2
   [junit4] Completed [247/454] on J1 in 3.12s, 4 tests
   [junit4]
   [junit4] Suite: org.apache.lucene.index.Test2BDocs
   [junit4] IGNOR/A 0.00s J1 | Test2BDocs.test2BDocs
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=Takes ~30min))
   [junit4] Completed [248/454] on J1 in 0.20s, 1 test, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnDiskFull
   [junit4] IGNOR/A 0.01s J1 | TestIndexWriterOnDiskFull.testAddIndexOnDiskFull
   [junit4]    > Assumption #1: This test cannot run with Memory codec
   [junit4] Completed [249/454] on J1 in 0.53s, 4 tests, 1 skipped
   [junit4]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpans
   [junit4] Completed [250/454] on J1 in 0.98s, 26 tests
   [junit

[...truncated too long message...]

space/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

common.resolve:

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/null675636717
     [copy] Copying 35 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/null675636717
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/null675636717

resolve-example:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/null285820233
     [copy] Copying 216 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/null285820233
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/null285820233

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 37ms :: artifacts dl 5ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 341 minutes 40 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 34 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/34/

9 tests failed.
FAILED:  org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates

Error Message:
_2_b.fnm

Stack Trace:
java.io.FileNotFoundException: _2_b.fnm
        at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77:81E61187C6FA095]:0)
        at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
        at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
        at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
        at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
        at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
        at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
        at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
        at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
        at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
        at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
        at org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestDuelingCodecsAtNight

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77]:0)


FAILED:  org.apache.lucene.index.TestNumericDocValuesUpdates.testTonsOfUpdates

Error Message:
_1_a.fnm

Stack Trace:
java.io.FileNotFoundException: _1_a.fnm
        at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77:81E61187C6FA095]:0)
        at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
        at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
        at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
        at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
        at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
        at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
        at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
        at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
        at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
        at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
        at org.apache.lucene.index.TestNumericDocValuesUpdates.testTonsOfUpdates(TestNumericDocValuesUpdates.java:1557)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([A55A6CA3FBAB0294]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([A55A6CA3FBAB0294]:0)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([CA97B3FC983C7403:42C38C2636C019FB]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertTrue(Assert.java:54)
        at org.apache.solr.cloud.MultiThreadedOCPTest.testFillWorkQueue(MultiThreadedOCPTest.java:111)
        at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:66)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
KeeperErrorCode = Session expired for /overseer_elect/leader

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
        at __randomizedtesting.SeedInfo.seed([CA97B3FC983C7403:42C38C2636C019FB]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderId(OverseerTaskProcessor.java:374)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderNode(OverseerTaskProcessor.java:365)
        at org.apache.solr.cloud.RollingRestartTest.waitUntilOverseerDesignateIsLeader(RollingRestartTest.java:137)
        at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
        at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([CA97B3FC983C7403:42C38C2636C019FB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 713 lines...]
   [junit4] Suite: org.apache.lucene.index.TestNumericDocValuesUpdates
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestNumericDocValuesUpdates -Dtests.method=testTonsOfUpdates -Dtests.seed=703BBF139E4F8F77 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=en-IE -Dtests.timezone=CTT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   23.7s J1 | TestNumericDocValuesUpdates.testTonsOfUpdates <<<
   [junit4]    > Throwable #1: java.io.FileNotFoundException: _1_a.fnm
   [junit4]    > at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77:81E61187C6FA095]:0)
   [junit4]    > at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
   [junit4]    > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]    > at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
   [junit4]    > at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
   [junit4]    > at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
   [junit4]    > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
   [junit4]    > at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
   [junit4]    > at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
   [junit4]    > at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
   [junit4]    > at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
   [junit4]    > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
   [junit4]    > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
   [junit4]    > at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
   [junit4]    > at org.apache.lucene.index.TestNumericDocValuesUpdates.testTonsOfUpdates(TestNumericDocValuesUpdates.java:1557)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/J1/temp/lucene.index.TestNumericDocValuesUpdates_703BBF139E4F8F77-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=en-IE, timezone=CTT
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=196414456,total=358088704
   [junit4]   2> NOTE: All tests run in this JVM: [TestBlockPostingsFormat3, TestByteArrayDataInput, TestBytesRefAttImpl, TestAutomatonQueryUnicode, TestDoc, TestTimeLimitingCollector, TestBooleanOr, TestBytesRefArray, TestDocumentsWriterStallControl, TestSearcherManager, TestPointValues, TestFixedBitDocIdSet, TestFilterSpans, TestMSBRadixSorter, TestComplexExplanations, TestFieldType, TestTransactions, TestTermdocPerf, TestTransactionRollback, TestDemo, TestPriorityQueue, TestFilterDirectoryReader, TestMixedDocValuesUpdates, TestInPlaceMergeSorter, TestDoubleValuesSource, TestQueryBuilder, TestPrefixCodedTerms, TestCodecUtil, TestSizeBoundedForceMerge, TestPolygon, TestNumericDocValuesUpdates]
   [junit4] Completed [134/454 (1!)] on J1 in 82.41s, 32 tests, 1 error <<< FAILURES!

[...truncated 839 lines...]
   [junit4] Suite: org.apache.lucene.index.TestBinaryDocValuesUpdates
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBinaryDocValuesUpdates -Dtests.method=testTonsOfUpdates -Dtests.seed=703BBF139E4F8F77 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=hu -Dtests.timezone=Asia/Ashkhabad -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   29.7s J2 | TestBinaryDocValuesUpdates.testTonsOfUpdates <<<
   [junit4]    > Throwable #1: java.io.FileNotFoundException: _2_b.fnm
   [junit4]    > at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77:81E61187C6FA095]:0)
   [junit4]    > at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
   [junit4]    > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]    > at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
   [junit4]    > at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
   [junit4]    > at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
   [junit4]    > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
   [junit4]    > at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
   [junit4]    > at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
   [junit4]    > at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
   [junit4]    > at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
   [junit4]    > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
   [junit4]    > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
   [junit4]    > at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
   [junit4]    > at org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> TEST: isNRT=false reader1=StandardDirectoryReader(segments_1:4 _0(7.0.0):c2)
   [junit4]   1> TEST: now reopen
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/J2/temp/lucene.index.TestBinaryDocValuesUpdates_703BBF139E4F8F77-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=hu, timezone=Asia/Ashkhabad
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=263830608,total=510656512
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntsRef, TestConjunctions, TestStressIndexing2, TestSearchForDuplicates, TestSameTokenSamePosition, TestVersion, TestUsageTrackingFilterCachingPolicy, TestShardSearching, TestLSBRadixSorter, TestCachingTokenFilter, TestTimSorterWorstCase, TestTermRangeQuery, TestDocValues, TestCompiledAutomaton, TestAutomaton, TestFilterWeight, TestCharsRefBuilder, TestRollingBuffer, TestSimpleExplanationsOfNonMatches, TestNRTThreads, TestPrefixQuery, TestStressAdvance, TestMultiFields, TestLucene60PointsFormat, TestSpanBoostQuery, TestSentinelIntSet, TestEarlyTerminatingSortingCollector, TestByteBlockPool, TestExceedMaxTermLength, TestHugeRamFile, TestDirectMonotonic, TestIOUtils, TestStopFilter, TestDemoParallelLeafReader, TestSetOnce, TestIndexWriterMergePolicy, TestIndexSearcher, TestSleepingLockWrapper, TestGeoEncodingUtils, TestLazyProxSkipping, TestAttributeSource, TestPayloads, TestIndexCommit, TestFilterDirectory, TestTopDocsCollector, TestRegexpQuery, TestConjunctionDISI, TermInSetQueryTest, TestSpanTermQuery, TestSearchWithThreads, TestReadOnlyIndex, TestDirectoryReader, TestLongPostings, TestTermVectors, Test2BSortedDocValuesOrds, TestBagOfPostings, TestPackedTokenAttributeImpl, TestField, TestSimpleFSDirectory, TestMultiMMap, TestSegmentReader, TestPackedInts, TestMixedCodecs, Test2BBKDPoints, TestDocValuesScoring, TestIndexWriterThreadsToSegments, TestDocument, TestQueryRescorer, TestFieldReuse, TestArrayUtil, TestRadixSelector, TestNumericUtils, TestTopDocsMerge, TestIndexManyDocuments, TestDisjunctionMaxQuery, TestConsistentFieldNumbers, TestMultiset, TestIndexWriterForceMerge, TestByteSlices, TestWeakIdentityMap, TestSimpleExplanations, TestPhraseQuery, TestCollectionUtil, TestBagOfPositions, TestFixedBitSet, TestFieldCacheRewriteMethod, TestPagedBytes, TestCrash, TestSegmentTermDocs, TestPerFieldPostingsFormat2, TestFieldsReader, TestDocValuesIndexing, TestSimilarityBase, TestDeterminizeLexicon, Test2BDocs, TestIndexWriterOnDiskFull, TestSpans, TestCustomNorms, TestFlex, TestUTF32ToUTF8, TestMultiPhraseQuery, TestDeterminism, TestOmitPositions, TestParallelLeafReader, TestStressNRT, TestIndexInput, TestSloppyPhraseQuery2, TestCodecHoldsOpenFiles, MultiCollectorTest, TestScoreCachingWrappingScorer, TestSimpleAttributeImpl, Test2BPositions, TestNewestSegment, Test2BNumericDocValues, Test2BBinaryDocValues, Test2BPagedBytes, TestCharArraySet, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestBlockPostingsFormat, TestBlockPostingsFormat2, TestForUtil, TestLucene50CompoundFormat, TestLucene50StoredFieldsFormatHighCompression, TestLucene60FieldInfoFormat, TestIndexedDISI, TestLucene70DocValuesFormat, TestLucene70SegmentInfoFormat, TestGeoUtils, TestPolygon2D, TestAllFilesCheckIndexHeader, TestAllFilesDetectTruncation, TestAllFilesHaveChecksumFooter, TestBinaryDocValuesUpdates]
   [junit4] Completed [337/454 (2!)] on J2 in 63.03s, 29 tests, 1 error <<< FAILURES!

[...truncated 622 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2> aug 25, 2017 6:58:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=7996, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[703BBF139E4F8F77], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:70)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2478)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:965)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=7995, name=SUITE-TestDuelingCodecsAtNight-seed#[703BBF139E4F8F77], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecsAtNight -Dtests.method=testBigEquals -Dtests.seed=703BBF139E4F8F77 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=sk-SK -Dtests.timezone=Asia/Srednekolymsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7149s J2 | TestDuelingCodecsAtNight.testBigEquals <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77]:0)
   [junit4]   2> aug 25, 2017 6:58:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> aug 25, 2017 6:59:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestDuelingCodecsAtNight:
   [junit4]   2>    1) Thread[id=7996, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[703BBF139E4F8F77], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at sun.nio.ch.NativeThread.current(Native Method)
   [junit4]   2>         at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:284)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:50)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
   [junit4]   2>         at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:171)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:140)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:59)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2478)
   [junit4]   2>         at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:965)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> aug 25, 2017 6:59:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=7996, name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[703BBF139E4F8F77], state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDuelingCodecsAtNight_703BBF139E4F8F77-001
   [junit4]   2> Aug 24, 2017 7:59:12 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {titleTokenized=DFR I(ne)L1, body=DFR G1}, locale=sk-SK, timezone=Asia/Srednekolymsk
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=281930680,total=526385152
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntsRef, TestConjunctions, TestStressIndexing2, TestSearchForDuplicates, TestSameTokenSamePosition, TestVersion, TestUsageTrackingFilterCachingPolicy, TestShardSearching, TestLSBRadixSorter, TestCachingTokenFilter, TestTimSorterWorstCase, TestTermRangeQuery, TestDocValues, TestCompiledAutomaton, TestAutomaton, TestFilterWeight, TestCharsRefBuilder, TestRollingBuffer, TestSimpleExplanationsOfNonMatches, TestNRTThreads, TestPrefixQuery, TestStressAdvance, TestMultiFields, TestLucene60PointsFormat, TestSpanBoostQuery, TestSentinelIntSet, TestEarlyTerminatingSortingCollector, TestByteBlockPool, TestExceedMaxTermLength, TestHugeRamFile, TestDirectMonotonic, TestIOUtils, TestStopFilter, TestDemoParallelLeafReader, TestSetOnce, TestIndexWriterMergePolicy, TestIndexSearcher, TestSleepingLockWrapper, TestGeoEncodingUtils, TestLazyProxSkipping, TestAttributeSource, TestPayloads, TestIndexCommit, TestFilterDirectory, TestTopDocsCollector, TestRegexpQuery, TestConjunctionDISI, TermInSetQueryTest, TestSpanTermQuery, TestSearchWithThreads, TestReadOnlyIndex, TestDirectoryReader, TestLongPostings, TestTermVectors, Test2BSortedDocValuesOrds, TestBagOfPostings, TestPackedTokenAttributeImpl, TestField, TestSimpleFSDirectory, TestMultiMMap, TestSegmentReader, TestPackedInts, TestMixedCodecs, Test2BBKDPoints, TestDocValuesScoring, TestIndexWriterThreadsToSegments, TestDocument, TestQueryRescorer, TestFieldReuse, TestArrayUtil, TestRadixSelector, TestNumericUtils, TestTopDocsMerge, TestIndexManyDocuments, TestDisjunctionMaxQuery, TestConsistentFieldNumbers, TestMultiset, TestIndexWriterForceMerge, TestByteSlices, TestWeakIdentityMap, TestSimpleExplanations, TestPhraseQuery, TestCollectionUtil, TestBagOfPositions, TestFixedBitSet, TestFieldCacheRewriteMethod, TestPagedBytes, TestCrash, TestSegmentTermDocs, TestPerFieldPostingsFormat2, TestFieldsReader, TestDocValuesIndexing, TestSimilarityBase, TestDeterminizeLexicon, Test2BDocs, TestIndexWriterOnDiskFull, TestSpans, TestCustomNorms, TestFlex, TestUTF32ToUTF8, TestMultiPhraseQuery, TestDeterminism, TestOmitPositions, TestParallelLeafReader, TestStressNRT, TestIndexInput, TestSloppyPhraseQuery2, TestCodecHoldsOpenFiles, MultiCollectorTest, TestScoreCachingWrappingScorer, TestSimpleAttributeImpl, Test2BPositions, TestNewestSegment, Test2BNumericDocValues, Test2BBinaryDocValues, Test2BPagedBytes, TestCharArraySet, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestBlockPostingsFormat, TestBlockPostingsFormat2, TestForUtil, TestLucene50CompoundFormat, TestLucene50StoredFieldsFormatHighCompression, TestLucene60FieldInfoFormat, TestIndexedDISI, TestLucene70DocValuesFormat, TestLucene70SegmentInfoFormat, TestGeoUtils, TestPolygon2D, TestAllFilesCheckIndexHeader, TestAllFilesDetectTruncation, TestAllFilesHaveChecksumFooter, TestBinaryDocValuesUpdates, TestDocsWithFieldSet, TestDuelingCodecsAtNight]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecsAtNight -Dtests.seed=703BBF139E4F8F77 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=sk-SK -Dtests.timezone=Asia/Srednekolymsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestDuelingCodecsAtNight (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([703BBF139E4F8F77]:0)
   [junit4] Completed [454/454 (3!)] on J2 in 7220.47s, 2 tests, 2 errors <<< FAILURES!

[...truncated 8132 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> أغس 24, 2017 9:26:05 م com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    2) Thread[id=13, name=SUITE-TestGeo3DPoint-seed#[A55A6CA3FBAB0294], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
   [junit4]   2>    3) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[A55A6CA3FBAB0294], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findCrossings(Plane.java:667)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$DualCrossingEdgeIterator.matches(GeoComplexPolygon.java:997)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Node.traverse(GeoComplexPolygon.java:487)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Node.traverse(GeoComplexPolygon.java:495)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Node.traverse(GeoComplexPolygon.java:495)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Tree.traverse(GeoComplexPolygon.java:583)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$Tree.traverse(GeoComplexPolygon.java:569)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:327)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:849)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=A55A6CA3FBAB0294 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-LY -Dtests.timezone=America/Guadeloupe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   6604s J0 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([A55A6CA3FBAB0294]:0)
   [junit4]   2> أغس 24, 2017 9:26:05 م com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> أغس 24, 2017 9:26:25 م com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[A55A6CA3FBAB0294], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:164)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:849)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> أغس 24, 2017 9:26:25 م com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[A55A6CA3FBAB0294], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> أغس 24, 2017 9:26:28 م com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[A55A6CA3FBAB0294], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:814)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1251)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:380)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{id=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1829, maxMBSortInHeap=5.480038349801, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-LY, timezone=America/Guadeloupe
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=24820280,total=429391872
   [junit4]   2> NOTE: All tests run in this JVM: [GeoCircleTest, TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=A55A6CA3FBAB0294 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-LY -Dtests.timezone=America/Guadeloupe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([A55A6CA3FBAB0294]:0)
   [junit4] Completed [11/11 (1!)] on J0 in 7223.43s, 12 tests, 2 errors <<< FAILURES!

[...truncated 1874 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_CA97B3FC983C7403-001/init-core-data-001
   [junit4]   2> 35484 WARN  (SUITE-RollingRestartTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 35484 INFO  (SUITE-RollingRestartTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 35486 INFO  (SUITE-RollingRestartTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 37129 INFO  (SUITE-RollingRestartTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /xcn/sm
   [junit4]   2> 37395 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 37405 INFO  (Thread-2) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 37421 INFO  (Thread-2) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 37996 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.ZkTestServer start zk server on port:42580
   [junit4]   2> 40462 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 40739 ERROR (Thread-2) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 43289 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 43352 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 43355 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 43386 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 43402 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 43421 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 43438 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 43441 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 43443 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 43468 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 43489 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 43754 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 47436 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 47695 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f1d5ffb{/xcn/sm,null,AVAILABLE}
   [junit4]   2> 47976 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@34d37122{SSL,[ssl, http/1.1]}{127.0.0.1:?????}
   [junit4]   2> 47977 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.e.j.s.Server Started @?????ms
   [junit4]   2> 47977 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_CA97B3FC983C7403-001/tempDir-001/control/data, hostContext=/xcn/sm, hostPort=46291, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_CA97B3FC983C7403-001/control-001/cores}
   [junit4]   2> 48203 ERROR (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 48203 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 48204 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 48204 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 48485 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-25T01:57:06.077Z
   [junit4]   2> 48499 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 48651 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 48685 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_CA97B3FC983C7403-001/control-001/solr.xml
   [junit4]   2> 48819 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@389b0570, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 49065 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42580/solr
   [junit4]   2> 51163 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 51210 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46291_xcn%2Fsm
   [junit4]   2> 51212 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.Overseer Overseer (id=98541647685156869-127.0.0.1:46291_xcn%2Fsm-n_0000000000) starting
   [junit4]   2> 51554 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46291_xcn%2Fsm
   [junit4]   2> 51759 INFO  (OverseerStateUpdate-98541647685156869-127.0.0.1:46291_xcn%2Fsm-n_0000000000) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 52949 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@389b0570
   [junit4]   2> 52950 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@389b0570
   [junit4]   2> 52950 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@389b0570
   [junit4]   2> 53094 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_CA97B3FC983C7403-001/control-001/cores
   [junit4]   2> 53897 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 54000 INFO  (TEST-RollingRestartTest.test-seed#[CA97B3FC983C7403]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42580/solr ready
   [junit4]   2> 57955 INFO  (qtp129135909-43) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46291_xcn%252Fsm&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 58470 INFO  (OverseerThreadFactory-20-thread-1-processing-n:127.0.0.1:46291_xcn%2Fsm) [n:127.0.0.1:46291_xcn%2Fsm    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 59

[...truncated too long message...]

f1
   [junit4]   2> 8 /solr/collections/collection1/state.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 13 /solr/clusterprops.json
   [junit4]   2> 13 /solr/clusterstate.json
   [junit4]   2> 10 /solr/collections/solrj_collection4/state.json
   [junit4]   2> 10 /solr/collections/solrj_collection3/state.json
   [junit4]   2> 9 /solr/collections/solrj_collection/state.json
   [junit4]   2> 8 /solr/collections/solrj_collection2/state.json
   [junit4]   2> 3 /solr/overseer_elect/election/98541939668877321-127.0.0.1:42817_o_t%2Fdp-n_0000000001
   [junit4]   2> 2 /solr/collections/solrj_collection/leader_elect/shard1/election/98541939668877316-core_node3-n_0000000000
   [junit4]   2> 2 /solr/collections/control_collection/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98541939668877316-127.0.0.1:43338_o_t%2Fdp-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 13 /solr/collections
   [junit4]   2> 8 /solr/live_nodes
   [junit4]   2> 5 /solr/overseer/collection-queue-work
   [junit4]   2> 4 /solr/overseer/queue
   [junit4]   2> 3 /solr/overseer/queue-work
   [junit4]   2>
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SharedFSAutoReplicaFailoverTest -Dtests.method=test -Dtests.seed=CA97B3FC983C7403 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-LB -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE  260s J0 | SharedFSAutoReplicaFailoverTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<2> but was:<3>
   [junit4]    > at __randomizedtesting.SeedInfo.seed([CA97B3FC983C7403:42C38C2636C019FB]:0)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 4753946 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 4754563 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA97B3FC983C7403]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 4754570 WARN  (66726135@qtp-529842432-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34441) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 4754570 WARN  (66726135@qtp-529842432-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34441) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 4754627 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:41912) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 4754627 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:41912) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1343968965-127.0.0.1-1503630661649 (Datanode Uuid d643c0ff-6f55-42f6-8124-c45a813de3ed) service to lucene2-us-west.apache.org/127.0.0.1:41912
   [junit4]   2> 4754644 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA97B3FC983C7403]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 4755332 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA97B3FC983C7403]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 4755339 WARN  (1853441009@qtp-1204426116-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38545) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 4755339 WARN  (1853441009@qtp-1204426116-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38545) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 4755392 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:41912) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1343968965-127.0.0.1-1503630661649 (Datanode Uuid aa7fd889-8a57-4483-ae60-1d6de6a93d35) service to lucene2-us-west.apache.org/127.0.0.1:41912
   [junit4]   2> 4755807 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA97B3FC983C7403]-worker) [    ] o.m.log Stopped HttpServer2$[hidden email]:0
   [junit4]   2> 4755807 WARN  (283490319@qtp-1680807669-1 - Acceptor0 HttpServer2$[hidden email]:37617) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 4755808 WARN  (283490319@qtp-1680807669-1 - Acceptor0 HttpServer2$[hidden email]:37617) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA97B3FC983C7403-001
   [junit4]   2> Aug 25, 2017 3:15:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-LB, timezone=America/Panama
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=227259720,total=521666560
   [junit4]   2> NOTE: All tests run in this JVM: [TestRandomFlRTGCloud, MoveReplicaHDFSUlogDirTest, TestFileDictionaryLookup, TestSolrQueryParser, UUIDFieldTest, SolrPluginUtilsTest, TestRawTransformer, RankQueryTest, PreAnalyzedUpdateProcessorTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestLMDirichletSimilarityFactory, TestSolrCloudWithSecureImpersonation, TestMinMaxOnMultiValuedField, TestStressLucene, CoreMergeIndexesAdminHandlerTest, TestLocalFSCloudBackupRestore, TestLazyCores, CdcrUpdateLogTest, ImplicitSnitchTest, TestXmlQParserPlugin, TestSQLHandler, TestSchemaNameResource, DistributedSuggestComponentTest, TestWriterPerf, TestReloadAndDeleteDocs, TestIndexingPerformance, BasicDistributedZkTest, ExternalFileFieldSortTest, DistributedVersionInfoTest, TestImpersonationWithHadoopAuth, TestBackupRepositoryFactory, TestCloudJSONFacetJoinDomain, SolrCLIZkUtilsTest, TestDocSet, TestFaceting, TestCrossCoreJoin, TestUseDocValuesAsStored2, FieldMutatingUpdateProcessorTest, SmileWriterTest, StatsReloadRaceTest, TestCSVResponseWriter, TestManagedSchemaAPI, WordBreakSolrSpellCheckerTest, PluginInfoTest, TestReplicationHandlerBackup, DefaultValueUpdateProcessorTest, TestBlobHandler, SolrCoreTest, TestPrepRecovery, TestRandomCollapseQParserPlugin, TestMultiValuedNumericRangeQuery, TestCloudManagedSchema, TestCustomStream, XmlUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, TestCustomSort, CurrencyFieldTypeTest, TestDownShardTolerantSearch, TestSolrDeletionPolicy1, TestNestedDocsSort, OpenExchangeRatesOrgProviderTest, HttpSolrCallGetCoreTest, DocExpirationUpdateProcessorFactoryTest, AtomicUpdateProcessorFactoryTest, TestRemoteStreaming, PKIAuthenticationIntegrationTest, TestManagedResource, TestJoin, TestStressLiveNodes, SSLMigrationTest, TestFastOutputStream, TestJsonRequest, TestRecoveryHdfs, TestFieldResource, VMParamsZkACLAndCredentialsProvidersTest, SharedFSAutoReplicaFailoverTest]
   [junit4] Completed [270/729 (3!)] on J0 in 279.90s, 1 test, 1 failure <<< FAILURES!

[...truncated 44892 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 35 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/35/

9 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates

Error Message:
Timeout while trying to assert number of documents @ http://127.0.0.1:41220/_uh/qh/source_collection_shard1_replica_n2/

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ http://127.0.0.1:41220/_uh/qh/source_collection_shard1_replica_n2/
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D:77D8619FD106DF1A]:0)
        at org.apache.solr.cloud.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:256)
        at org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:236)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<10> but was:<20>
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:247)
        ... 42 more


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:     1) Thread[id=56439, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    2) Thread[id=56438, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
   1) Thread[id=56439, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   2) Thread[id=56438, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=56439, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.lang.Throwable.fillInStackTrace(Native Method)         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)         at java.lang.Throwable.<init>(Throwable.java:265)         at java.lang.Exception.<init>(Exception.java:66)         at java.lang.RuntimeException.<init>(RuntimeException.java:62)         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)         at org.apache.http.util.Asserts.check(Asserts.java:34)         at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)         at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)         at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)         at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)         at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)         at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)         at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    2) Thread[id=56438, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]         at java.security.AccessController.doPrivileged(Native Method)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)         at java.io.PrintStream.flush(PrintStream.java:338)         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at java.io.PrintStream.write(PrintStream.java:482)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)         at java.io.PrintStream.newLine(PrintStream.java:546)         at java.io.PrintStream.println(PrintStream.java:824)         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)         at java.lang.Throwable.printStackTrace(Throwable.java:658)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=56439, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.lang.Throwable.fillInStackTrace(Native Method)
        at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
        at java.lang.Throwable.<init>(Throwable.java:265)
        at java.lang.Exception.<init>(Exception.java:66)
        at java.lang.RuntimeException.<init>(RuntimeException.java:62)
        at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
        at org.apache.http.util.Asserts.check(Asserts.java:34)
        at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)
        at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   2) Thread[id=56438, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
        at java.security.AccessController.doPrivileged(Native Method)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
        at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.write(PrintStream.java:482)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
        at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
        at java.io.PrintStream.newLine(PrintStream.java:546)
        at java.io.PrintStream.println(PrintStream.java:824)
        at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
        at java.lang.Throwable.printStackTrace(Throwable.java:658)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)


FAILED:  org.apache.solr.cloud.ShardSplitTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1114)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:902)
        at org.apache.solr.cloud.ShardSplitTest.splitByRouteFieldTest(ShardSplitTest.java:691)
        at org.apache.solr.cloud.ShardSplitTest.test(ShardSplitTest.java:101)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
        ... 49 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=3410, name=testExecutor-374-thread-2, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3410, name=testExecutor-374-thread-2, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33637
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33637
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
        ... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        ... 9 more


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10219 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> ago 28, 2017 9:04:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:317)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testGeo3DRelations -Dtests.seed=37424D69F4F2ABF5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=es-CL -Dtests.timezone=America/Menominee -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   22.1s J1 | TestGeo3DPoint.testGeo3DRelations <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
   [junit4]   2> ago 28, 2017 9:04:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> ago 28, 2017 9:04:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:283)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ago 28, 2017 9:04:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> ago 28, 2017 9:04:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:274)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Direct), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1776, maxMBSortInHeap=5.016577876027453, sim=RandomSimilarity(queryNorm=false): {}, locale=es-CL, timezone=America/Menominee
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=69693584,total=497549312
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=37424D69F4F2ABF5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=es-CL -Dtests.timezone=America/Menominee -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
   [junit4] Completed [10/11 (1!)] on J1 in 7225.21s, 7 tests, 2 errors <<< FAILURES!

[...truncated 1813 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/init-core-data-001
   [junit4]   2> 346493 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 346493 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 346495 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 346495 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 350734 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 351462 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 351576 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 351833 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 352181 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_33969_hdfs____.w0x9h6/webapp
   [junit4]   2> 354830 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Started HttpServer2$[hidden email]:33969
   [junit4]   2> 358127 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 358146 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 358209 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36777_datanode____m4ojjv/webapp
   [junit4]   2> 359558 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36777
   [junit4]   2> 360999 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 361003 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 361150 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45430_datanode____.pgn1yh/webapp
   [junit4]   2> 363500 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45430
   [junit4]   2> 369252 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:34196) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 369254 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:34196) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 370534 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xe5b70e0ed6e77: from storage DS-bc1a05f6-6b17-4021-95a4-75d67f297135 node DatanodeRegistration(127.0.0.1:38376, datanodeUuid=9b5f89cc-19dd-4229-bd7e-03509dcd9cc9, infoPort=39607, infoSecurePort=0, ipcPort=45701, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: true, processing time: 54 msecs
   [junit4]   2> 370551 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xe5b70e1cbe78b: from storage DS-1bbdbdf0-39a2-412c-83de-78e0f07a5722 node DatanodeRegistration(127.0.0.1:32906, datanodeUuid=6748d72e-58dc-436a-9999-affbba7af3d3, infoPort=38852, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: true, processing time: 16 msecs
   [junit4]   2> 370551 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xe5b70e0ed6e77: from storage DS-c6bb5358-3062-4aa3-82c2-b07e34f756a3 node DatanodeRegistration(127.0.0.1:38376, datanodeUuid=9b5f89cc-19dd-4229-bd7e-03509dcd9cc9, infoPort=39607, infoSecurePort=0, ipcPort=45701, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 370551 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xe5b70e1cbe78b: from storage DS-e41833da-b85b-417c-ad14-23eb320a1a3b node DatanodeRegistration(127.0.0.1:32906, datanodeUuid=6748d72e-58dc-436a-9999-affbba7af3d3, infoPort=38852, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 373277 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 373281 INFO  (Thread-267) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 373281 INFO  (Thread-267) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 373381 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.ZkTestServer start zk server on port:33225
   [junit4]   2> 373428 ERROR (Thread-267) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 373711 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 373741 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 373742 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 373743 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 373744 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 373822 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 373903 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 373904 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 373905 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 373906 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 373923 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 373997 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 375053 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 375194 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2df280ba{/,null,AVAILABLE}
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6f51a86a{HTTP/1.1,[http/1.1]}{127.0.0.1:37022}
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.Server Started @386590ms
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:34196/hdfs__lucene2-us-west.apache.org_34196__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=37022, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores}
   [junit4]   2> 375195 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 375195 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-28T14:51:25.658Z
   [junit4]   2> 375291 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 375292 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/solr.xml
   [junit4]   2> 375386 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 375429 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33225/solr
   [junit4]   2> 376085 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 376086 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37022_
   [junit4]   2> 376135 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.c.Overseer Overseer (id=98561679939469316-127.0.0.1:37022_-n_0000000000) starting
   [junit4]   2> 376171 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37022_
   [junit4]   2> 376354 INFO  (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 377014 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
   [junit4]   2> 377110 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
   [junit4]   2> 377111 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
   [junit4]   2> 377112 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores
   [junit4]   2> 377385 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 377385 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33225/solr ready
   [junit4]   2> 377698 INFO  (qtp2134611594-1578) [n:127.0.0.1:37022_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37022_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 377845 INFO  (OverseerThreadFactory-295-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 378327 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 378327 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 378615 INFO  (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 379715 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 380039 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 380694 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 380810 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 380810 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
   [junit4]   2> 380912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home
   [junit4]   2> 380912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 380912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 380912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/]
   [junit4]   2> 380942 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
   [junit4]   2> 381058 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 381058 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 381058 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 383460 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 383492 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data
   [junit4]   2> 383669 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/index
   [junit4]   2> 383710 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 383710 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 383710 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 383795 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 383797 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1872306703, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 385109 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:32906 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bc1a05f6-6b17-4021-95a4-75d67f297135:NORMAL:127.0.0.1:38376|RBW], ReplicaUC[[DISK]DS-e41833da-b85b-417c-ad14-23eb320a1a3b:NORMAL:127.0.0.1:32906|RBW]]} size 0
   [junit4]   2> 385146 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38376 is added to blk_1073741825_1001 size 72
   [junit4]   2> 385288 WARN  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 385496 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 385496 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 385496 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 385576 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 385576 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 385578 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 385910 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2733c020[control_collection_shard1_replica_n1] main]
   [junit4]   2> 385912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 385912 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 385914 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 385952 INFO  (searcherExecutor-298-thread-1-processing-n:127.0.0.1:37022_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2733c020[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 385952 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576986892215255040
   [junit4]   2> 385983 INFO  (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37022/control_collection_shard1_replica_n1/
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37022/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 385984 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 386003 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37022/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 386105 INFO  (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 386154 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 386155 INFO  (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7828
   [junit4]   2> 386169 INFO  (qtp2134611594-1578) [n:127.0.0.1:37022_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 386260 INFO  (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 387175 INFO  (qtp2134611594-1578) [n:127.0.0.1:37022_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37022_&wt=javabin&version=2} status=0 QTime=9477
   [junit4]   2> 387248 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 387249 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33225/solr ready
   [junit4]   2> 387249 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 387250 INFO  (qtp2134611594-1582) [n:127.0.0.1:37022_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 387266 INFO  (OverseerThreadFactory-295-thread-2-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 387266 INFO  (OverseerCollectionConfigSetProcessor-98561679939469316-127.0.0.1:37022_-n_0000000000) [n:127.0.0.1:37022_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 387267 WARN  (OverseerThreadFactory-295-thread-2-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 387505 INFO  (qtp2134611594-1582) [n:127.0.0.1:37022_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 387506 INFO  (qtp2134611594-1582) [n:127.0.0.1:37022_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=255
   [junit4]   2> 388370 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/shard-1-001 of type NRT
   [junit4]   2> 388373 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 388397 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67abdb7b{/,null,AVAILABLE}
   [junit4]   2> 388397 INFO  (TEST-HdfsUnloadDistribut

[...truncated too long message...]

4D131819495438D]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1490372361
   [junit4]   2> 4893370 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 4893370 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@234e09e0: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@11e06129
   [junit4]   2> 4893390 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 4893390 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4dcf3683: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3627dfe9
   [junit4]   2> 4893402 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 4893402 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@514813b1: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@46799fab
   [junit4]   2> 4893402 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 4893434 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.c.Overseer Overseer (id=98561974902784028-127.0.0.1:40483__uh%2Fqh-n_0000000002) closing
   [junit4]   2> 4893434 INFO  (OverseerStateUpdate-98561974902784028-127.0.0.1:40483__uh%2Fqh-n_0000000002) [n:127.0.0.1:40483__uh%2Fqh    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40483__uh%2Fqh
   [junit4]   2> 4894388 WARN  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 4894389 INFO  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.c.SolrCore [source_collection_shard1_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e086f06
   [junit4]   2> 4894389 INFO  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.source_collection.shard1.replica_n2, tag=1309175558
   [junit4]   2> 4894389 INFO  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1f0ef195: rootName = null, domain = solr.core.source_collection.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.source_collection.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@66e1c78d
   [junit4]   2> 4894417 INFO  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.source_collection.shard1.leader, tag=1309175558
   [junit4]   2> 4894429 INFO  (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ source_collection:shard1
   [junit4]   2> 4894433 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5bb2f8b1{HTTP/1.1,[http/1.1]}{127.0.0.1:40483}
   [junit4]   2> 4894433 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7b27c0ca{/_uh/qh,null,UNAVAILABLE}
   [junit4]   2> 4894441 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testPartialReplication
   [junit4]   2> 4894441 ERROR (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4894442 INFO  (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37066 37066
   [junit4]   2> 4894578 INFO  (Thread-26610) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37066 37066
   [junit4]   2> 4894578 WARN  (Thread-26610) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 22 /solr/aliases.json
   [junit4]   2> 5 /solr/configs/conf1
   [junit4]   2> 4 /solr/collections/source_collection/leaders/shard1/leader
   [junit4]   2> 3 /solr/security.json
   [junit4]   2> 2 /solr/collections/tmp_collection/leaders/shard1/leader
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 22 /solr/clusterstate.json
   [junit4]   2> 22 /solr/clusterprops.json
   [junit4]   2> 4 /solr/collections/source_collection/cdcr/state/buffer
   [junit4]   2> 4 /solr/collections/source_collection/cdcr/state/process
   [junit4]   2> 3 /solr/collections/source_collection/state.json
   [junit4]   2> 2 /solr/collections/tmp_collection/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98561974902784004-127.0.0.1:41887__uh%2Fqh-n_0000000000
   [junit4]   2> 2 /solr/collections/source_collection/leader_elect/shard1/election/98561974902784004-core_node2-n_0000000000
   [junit4]   2> 2 /solr/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 2 /solr/collections/tmp_collection/cdcr/state/process
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 22 /solr/collections
   [junit4]   2> 21 /solr/live_nodes
   [junit4]   2> 2 /solr/overseer/queue
   [junit4]   2> 2 /solr/overseer/collection-queue-work
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CdcrReplicationHandlerTest_A4D131819495438D-001
   [junit4]   2> Aug 28, 2017 4:06:44 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=FSTOrd50}, docValues:{_version_=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=918, maxMBSortInHeap=6.724546786203957, sim=RandomSimilarity(queryNorm=true): {}, locale=pt-BR, timezone=Europe/Bratislava
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=171909696,total=525336576
   [junit4]   2> NOTE: All tests run in this JVM: [DirectoryFactoryTest, DeleteReplicaTest, V2ApiIntegrationTest, TestGraphMLResponseWriter, ShardSplitTest, TestFieldCacheSort, DistributedDebugComponentTest, TestFoldingMultitermQuery, TestCloudPseudoReturnFields, HdfsChaosMonkeyNothingIsSafeTest, BadIndexSchemaTest, TestRestoreCore, TestHighFrequencyDictionaryFactory, TestTolerantSearch, TestChildDocTransformer, ConjunctionSolrSpellCheckerTest, DeleteInactiveReplicaTest, CloneFieldUpdateProcessorFactoryTest, ReplaceNodeTest, JSONWriterTest, TestIntervalFaceting, MultiTermTest, TestFieldCollectionResource, TestStressReorder, TestStressInPlaceUpdates, SearchHandlerTest, TestOnReconnectListenerSupport, HdfsRestartWhileUpdatingTest, TestRestManager, ExplicitHLLTest, TestExactStatsCache, TestDistribDocBasedVersion, PrimitiveFieldTypeTest, DocValuesMissingTest, DateFieldTest, LeaderFailoverAfterPartitionTest, DistributedQueueTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestPointFields, SpatialHeatmapFacetsTest, TestDocTermOrds, TestNonDefinedSimilarityFactory, SimpleCollectionCreateDeleteTest, DocumentBuilderTest, HdfsNNFailoverTest, SolrJmxReporterTest, MinimalSchemaTest, StatsComponentTest, TestRawResponseWriter, DistributedFacetPivotSmallTest, SearchHandlerTest, ConnectionManagerTest, FullSolrCloudDistribCmdsTest, HdfsTlogReplayBufferedWhileIndexingTest, DistribCursorPagingTest, TestDelegationWithHadoopAuth, LeaderElectionTest, TestFieldCacheWithThreads, ChangedSchemaMergeTest, RegexBoostProcessorTest, RollingRestartTest, CleanupOldIndexTest, DirectUpdateHandlerOptimizeTest, OverriddenZkACLAndCredentialsProvidersTest, TestDFRSimilarityFactory, DateMathParserTest, CollectionStateFormat2Test, CdcrReplicationHandlerTest]
   [junit4] Completed [676/729 (5!)] on J2 in 756.95s, 5 tests, 1 failure <<< FAILURES!

[...truncated 43486 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 36 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/36/

5 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
        at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:F7FCCE7C510D8FD5]:0)
        at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1267)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster

Error Message:
Document mismatch on target after sync expected:<20000> but was:<10100>

Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<20000> but was:<10100>
        at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:ABEDBAFF18A751D6]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster(CdcrBootstrapTest.java:309)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:F7FCCE7C510D8FD5]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10149 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> ??? 01, 2017 5:51:40 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.addPoint(Plane.java:1953)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1418)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:423)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoCompositeMembershipShape.getBounds(GeoCompositeMembershipShape.java:75)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=23DB77BF5E4E778A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-OM -Dtests.timezone=Pacific/Guam -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7196s J1 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
   [junit4]   2> ??? 01, 2017 5:51:40 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> ??? 01, 2017 5:52:00 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.StandardXYZSolid.<init>(StandardXYZSolid.java:153)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZSolidFactory.makeXYZSolid(XYZSolidFactory.java:64)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:847)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ??? 01, 2017 5:52:00 ? com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/spatial3d/test/J1/temp/lucene.spatial3d.TestGeo3DPoint_23DB77BF5E4E778A-001
   [junit4]   2> Aug 31, 2017 7:52:02 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Direct), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1940, maxMBSortInHeap=7.655887576976247, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-OM, timezone=Pacific/Guam
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=180741752,total=484442112
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=23DB77BF5E4E778A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-OM -Dtests.timezone=Pacific/Guam -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
   [junit4] Completed [11/11 (1!)] on J1 in 7223.21s, 4 tests, 2 errors <<< FAILURES!

[...truncated 2778 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/init-core-data-001
   [junit4]   2> 5301577 WARN  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 5301577 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 5301578 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 5301579 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001
   [junit4]   2> 5301579 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 5301589 INFO  (Thread-73111) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 5301589 INFO  (Thread-73111) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 5301646 ERROR (Thread-73111) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 5301689 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:33395
   [junit4]   2> 5301799 INFO  (jetty-launcher-1061-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 5301799 INFO  (jetty-launcher-1061-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 5301874 INFO  (jetty-launcher-1061-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31b3a1ee{/solr,null,AVAILABLE}
   [junit4]   2> 5301875 INFO  (jetty-launcher-1061-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7999b4d2{HTTP/1.1,[http/1.1]}{127.0.0.1:35283}
   [junit4]   2> 5301875 INFO  (jetty-launcher-1061-thread-1) [    ] o.e.j.s.Server Started @5314197ms
   [junit4]   2> 5301875 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35283}
   [junit4]   2> 5301875 ERROR (jetty-launcher-1061-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 5301960 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 5301960 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 5301960 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 5301960 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-31T21:42:36.942Z
   [junit4]   2> 5301955 INFO  (jetty-launcher-1061-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c467e30{/solr,null,AVAILABLE}
   [junit4]   2> 5302024 INFO  (jetty-launcher-1061-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@473fc035{HTTP/1.1,[http/1.1]}{127.0.0.1:44516}
   [junit4]   2> 5302024 INFO  (jetty-launcher-1061-thread-2) [    ] o.e.j.s.Server Started @5314347ms
   [junit4]   2> 5302024 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44516}
   [junit4]   2> 5302025 ERROR (jetty-launcher-1061-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 5302025 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 5302025 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 5302025 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 5302025 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-31T21:42:37.007Z
   [junit4]   2> 5302118 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5302119 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5302171 INFO  (jetty-launcher-1061-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33395/solr
   [junit4]   2> 5302218 INFO  (jetty-launcher-1061-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33395/solr
   [junit4]   2> 5302647 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5302711 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44516_solr
   [junit4]   2> 5302712 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.c.Overseer Overseer (id=98580283825324037-127.0.0.1:44516_solr-n_0000000000) starting
   [junit4]   2> 5302761 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44516_solr
   [junit4]   2> 5302938 INFO  (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5303040 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5303042 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5303057 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35283_solr
   [junit4]   2> 5303129 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 5303133 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 5303259 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303294 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303294 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303295 INFO  (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node2/.
   [junit4]   2> 5303559 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303590 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303590 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5303591 INFO  (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node1/.
   [junit4]   2> 5303894 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 5303895 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33395/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 5304078 WARN  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 5304226 WARN  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 5304243 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 5304325 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_35259_hdfs____.tkjr36/webapp
   [junit4]   2> 5305247 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Started HttpServer2$[hidden email]:35259
   [junit4]   2> 5305712 WARN  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 5305713 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 5305785 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35554_datanode____bzkm3d/webapp
   [junit4]   2> 5306820 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35554
   [junit4]   2> 5307289 WARN  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 5307290 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 5307369 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_44438_datanode____f6yd1c/webapp
   [junit4]   2> 5308449 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:35262) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 5308518 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xf5da17843cc6d: from storage DS-0995900b-2a7f-4b81-959e-495da6b15206 node DatanodeRegistration(127.0.0.1:43963, datanodeUuid=3f58a021-348b-4320-93b0-377ff6d78662, infoPort=40459, infoSecurePort=0, ipcPort=40659, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: true, processing time: 33 msecs
   [junit4]   2> 5308526 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xf5da17843cc6d: from storage DS-6330f322-6957-47ec-8012-2b0ef064fb59 node DatanodeRegistration(127.0.0.1:43963, datanodeUuid=3f58a021-348b-4320-93b0-377ff6d78662, infoPort=40459, infoSecurePort=0, ipcPort=40659, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 5308636 INFO  (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44438
   [junit4]   2> 5309635 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:35262) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 5309716 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xf5da1c118e971: from storage DS-32006cea-ed1b-4f2d-8150-815254924387 node DatanodeRegistration(127.0.0.1:44101, datanodeUuid=1205791d-58a4-428f-b8a0-2b1789d39e16, infoPort=34593, infoSecurePort=0, ipcPort=45338, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: true, processing time: 8 msecs
   [junit4]   2> 5309716 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0xf5da1c118e971: from storage DS-fc89b135-57f2-4ed5-90df-23bb75c813d5 node DatanodeRegistration(127.0.0.1:44101, datanodeUuid=1205791d-58a4-428f-b8a0-2b1789d39e16, infoPort=34593, infoSecurePort=0, ipcPort=45338, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 5310131 INFO  (TEST-HdfsRecoveryZkTest.test-seed#[7FA8F1A6FFF1E22D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 5310163 INFO  (qtp1982336695-79706) [n:127.0.0.1:35283_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5310174 INFO  (OverseerThreadFactory-2947-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
   [junit4]   2> 5310314 INFO  (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"recoverytest",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"recoverytest_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:35283/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 5310377 INFO  (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"recoverytest",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"recoverytest_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:44516/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 5310751 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 5310752 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 5310817 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 5310818 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 5310939 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5310939 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5310939 INFO  (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5310954 INFO  (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5311830 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 5311891 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 5311906 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5311906 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using configuration from collection recoverytest, trusted=true
   [junit4]   2> 5311921 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.core.recoverytest.shard1.replica_n1' (registry 'solr.core.recoverytest.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5311954 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:35262/data
   [junit4]   2> 5311954 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 5311954 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5311954 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node1/recoverytest_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/]
   [junit4]   2> 5311955 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 5312035 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 5312035 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 5312035 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 5312038 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 5312086 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 5312140 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5312140 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using configuration from collection recoverytest, trusted=true
   [junit4]   2> 5312148 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.core.recoverytest.shard1.replica_n2' (registry 'solr.core.recoverytest.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
   [junit4]   2> 5312149 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:35262/data
   [junit4]   2> 5312149 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 5312149 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5312149 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node2/recoverytest_shard1_replica_n2], dataDir=[hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/]
   [junit4]   2> 5312150 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 5312151 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data
   [junit4]   2> 5312151 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 5312183 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 5312183 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 5312183 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 5312624 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 5312626 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data
   [junit4]   2> 5312750 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/index
   [junit4]   2> 5312784 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 5312784 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 5312784 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 5312860 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/index
   [junit4]   2> 5312877 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 5312877 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 5312877 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 5312928 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 5312946 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 5313331 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44101 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|RBW], ReplicaUC[[DISK]DS-fc89b135-57f2-4ed5-90df-23bb75c813d5:NORMAL:127.0.0.1:44101|RBW]]} size 0
   [junit4]   2> 5313378 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43963 is added to blk_1073741825_1001 size 72
   [junit4]   2> 5313378 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43963 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-32006cea-ed1b-4f2d-8150-815254924387:NORMAL:127.0.0.1:44101|RBW], ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|FINALIZED]]} size 0
   [junit4]   2> 5313378 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44101 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-32006cea-ed1b-4f2d-8150-815254924387:NORMAL:127.0.0.1:44101|RBW], ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|FINALIZED]]} size 0
   [junit4]   2> 5313798 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 5313798 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5313798 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 5313832 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5313832 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5313846 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 5313846 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5313847 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 5314022 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5314022 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5314074 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@72ae5b47[recoverytest_shard1_replica_n2] main]
   [junit4]   2> 5314075 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 5314075 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 5314076 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 5314076 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577284554255761408
   [junit4]   2> 5314091 INFO  (searcherExecutor-2953-thread-1-processing-n:127.0.0.1:44516_solr x:recoverytest_shard1_replica_n2 s:shard1 c:recoverytest r:core_node2) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [recoverytest_shard1_replica_n2] Registered new searcher Searcher@72ae5b47[recoverytest_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5314092 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@274525a2[recoverytest_shard1_replica_n1] main]
   [junit4]   2> 5314093 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 5314093 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 5314093 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 5314094 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577284554274635776
   [junit4]   2> 5314095 INFO  (searcherExecutor-2952-thread-1-processing-n:127.0.0.1:35283_solr x:recoverytest_shard1_replica_n1 s:shard1 c:recoverytest r:core_node1) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [recoverytest_shard1_replica_n1] Registered new searcher Searcher@274525a2[recoverytest_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5314156 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 5314156 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 5314156 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/
   [junit4]   2> 5314156 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 url=http://127.0.0.1:35283/solr START replicas=[http://127.0.0.1:44516/solr/recoverytest_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 5314156 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 url=http://127.0.0.1:35283/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 5314157 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314157 INFO  (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314158 INFO  (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314158 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314192 INFO  (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.S.Request [recoverytest_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=19
   [junit4]   2> 5314204 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 5314204 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 5314204 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 5314206 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/ shard1
   [junit4]   2> 5314342 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314342 INFO  (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314342 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314342 INFO  (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314370 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 5314375 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3623
   [junit4]   2> 5314493 INFO  (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314493 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314493 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5314493 INFO  (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5315144 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4327
   [junit4]   2> 5315161 INFO  (qtp1982336695-79706) [n:127.0.0.1:35283_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 5315305 INFO  (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5315305 INFO  (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5315309 INFO  (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5315309 INFO  (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5316167 INFO  (qtp1982336695-79706) [n:127.0.0.1:35283_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=6003
   [junit4]   2> 5316187 INFO  (TEST-HdfsRecoveryZkTest.test-seed#[7FA8F1A6FFF1E22D]) [    ] o.a.s.c.RecoveryZkTest Indexing 3000 documents
   [junit4]   2> 5316219 INFO  (OverseerCollectionConfigSetProcessor-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 5316455 DEBUG (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000 refcount=1}
   [junit4]   2> 5316630 DEBUG (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000 refcount=1}
   [junit4]   2> 5316825 INFO  (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-0 (1577284556768149504)]} 0 259
   [junit4]   2> 5316825 INFO  (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-0 (1577284556606668800)]} 0 325
   [junit4]   2> 5316826 INFO  (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1577284556606668800)]} 0 507
   [junit4]   2> 5316826 INFO  (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1577284556768149504)]} 0 354
   [junit4]   2> 5316849 INFO  (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-1 (1577284557139345408)]} 0 22
   [junit4]   2> 5316849 INFO  (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-1 (1577284557140393984)]} 0 21
   [junit4]   2> 5316857 INFO  (qtp1982336695-79710) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1577284557139345408)]} 0 31
   [junit4]   2> 5316857 INFO  (qtp1982336695-79709) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1577284557140393984)]} 0 30
   [junit4]   2> 5316882 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-2 (1577284557172899840)]} 0 8
   [junit4]   2> 5316882 INFO  (qtp1982336695-79703) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1577284557172899840)]} 0 24
   [junit4]   2> 5316882 INFO  (qtp593564255-79718) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-2 (1577284557181288448)]} 0 8
   [junit4]   2> 5316882 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1577284557181288448)]} 0 16
   [junit4]   2> 5316916 INFO  (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-0 (-1577284557214842880)]} 0 0
   [junit4]   2> 5316927 INFO  (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-0 (-1577284557214842881)]} 0 12
   [junit4]   2> 5316927 INFO  (qtp1982336695-79706) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1577284557214842880)]} 0 29
   [junit4]   2> 5316931 INFO  (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1577284557214842881)]} 0 32
   [junit4]   2> 5316932 INFO  (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-3 (1577284557249445888)]} 0 0
   [junit4]   2> 5316932 INFO  (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1577284557249445888)]} 0 1
   [junit4]   2> 5316950 INFO  (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-1 (-1577284557250494464)]} 0 1
   [junit4]   2> 5316950 INFO  (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1577284557250494464)]} 0 17
   [junit4]   2> 5316963 INFO  (qtp593564255-79718) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-4 (1577284557269368832)]} 0 12
   [junit4]   2> 5316963 INFO  (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-3 (1577284557266223104)]} 0 14
   [junit4]   2> 5316964 INFO  (qtp1982336695-79710) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1577284557266223104)]} 0 16
   [junit4]   2> 5316978 INFO  (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-1 (-1577284557296631808)]} 0 1
   [junit4]   2> 5316978 INFO  (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1577284557296631808)]} 0 2
   [junit4]   2> 5316979 INFO  (qtp1982336695-79703) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1577284557269368832)]} 0 29
   [junit4]   2> 5317024 INFO  (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-4 (1577284557298728960)]} 0 45
   [junit4]   2> 5317040 INFO  (qtp1982336695-79709) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1577284557298728960)]} 0 62
   [junit4]   2> 5317041 INFO  (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-4 (-1577284557300826112)]} 0 0
   [junit4]   2> 5317042 INFO  (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-4 (-1577284557365837824)]} 0 0
   [junit4]   2> 5317042 INFO  (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1577284557365837824)]} 0 1
   [junit4]   2> 5317044 INFO  (qtp1982336695-79706) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  webapp=/solr path=/update

[...truncated too long message...]

utoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 8057040 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 8057062 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 8057163 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 8057163 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-601873968-127.0.0.1-1504218268494 (Datanode Uuid 383bbb35-e0e6-46b7-9d07-18524b62d8cc) service to lucene2-us-west.apache.org/127.0.0.1:37596
   [junit4]   2> 8057164 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 8057265 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 8057366 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 8057366 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-601873968-127.0.0.1-1504218268494 (Datanode Uuid 0ebdb639-c99b-4b07-b1e8-8cb6a76f5152) service to lucene2-us-west.apache.org/127.0.0.1:37596
   [junit4]   2> 8057411 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [    ] o.m.log Stopped HttpServer2$[hidden email]:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001
   [junit4]   2> Aug 31, 2017 10:28:32 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1489, maxMBSortInHeap=7.226615572480204, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-DZ, timezone=America/Edmonton
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=108331760,total=522715136
   [junit4]   2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, BitVectorTest, TestMaxScoreQueryParser, TestDFISimilarityFactory, DirectSolrSpellCheckerTest, TestRequestForwarding, IndexSchemaRuntimeFieldTest, HdfsThreadLeakTest, DistributedFacetExistsSmallTest, LoggingHandlerTest, ZkControllerTest, VersionInfoTest, AddBlockUpdateTest, DOMUtilTest, TlogReplayBufferedWhileIndexingTest, CacheHeaderTest, SoftAutoCommitTest, SliceStateTest, TestStressVersions, TestConfigSetsAPIExclusivity, ScriptEngineTest, DocValuesNotIndexedTest, TestLRUCache, TestSolrCloudWithDelegationTokens, TestAtomicUpdateErrorCases, ResourceLoaderTest, TestJsonFacetRefinement, FileBasedSpellCheckerTest, OverseerTaskQueueTest, TestReqParamsAPI, SpatialRPTFieldTypeTest, CheckHdfsIndexTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistributedMLTComponentTest, SortSpecParsingTest, ShardRoutingCustomTest, TestFiltering, BasicFunctionalityTest, BasicAuthIntegrationTest, SolrCoreMetricManagerTest, TestSha256AuthenticationProvider, TestManagedSynonymFilterFactory, TestConfigSets, DataDrivenBlockJoinTest, TestInPlaceUpdatesDistrib, TestCharFilters, TestDynamicFieldResource, SolrJmxReporterCloudTest, BlockJoinFacetDistribTest, TestValueSourceCache, TestSearchPerf, TestSchemaResource, TestSolrIndexConfig, StatelessScriptUpdateProcessorFactoryTest, DocValuesTest, TestApiFramework, SystemInfoHandlerTest, TestFieldCacheVsDocValues, TestMiniSolrCloudClusterSSL, SpatialFilterTest, TestTrieFacet, TestSmileRequest, JsonLoaderTest, TestDeleteCollectionOnDownNodes, FileUtilsTest, TestManagedResourceStorage, SchemaVersionSpecificBehaviorTest, MoveReplicaTest, DistributedIntervalFacetingTest, AutoScalingHandlerTest, FacetPivotSmallTest, DeleteStatusTest, TestHashQParserPlugin, TestSimpleTrackingShardHandler, HdfsBasicDistributedZkTest, SuggestComponentTest, ConvertedLegacyTest, ResponseHeaderTest, HighlighterTest, RuleEngineTest, TestPartialUpdateDeduplication, RemoteQueryErrorTest, CoreAdminHandlerTest, HdfsRecoveryZkTest, DistributedQueryComponentCustomSortTest, CoreSorterTest, CustomCollectionTest, TestFuzzyAnalyzedSuggestions, CSVRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest, TestClusterProperties, PrimUtilsTest, TestClassNameShortening, TestComplexPhraseLeadingWildcard, SOLR749Test, TestEmbeddedSolrServerSchemaAPI, TestPerFieldSimilarity, TestSystemCollAutoCreate, ShufflingReplicaListTransformerTest, MissingSegmentRecoveryTest, TestBM25SimilarityFactory, TestLegacyNumericUtils, TestLockTree, HdfsDirectoryFactoryTest, BlockDirectoryTest, TestSweetSpotSimilarityFactory, TestSlowCompositeReaderWrapper, CloudMLTQParserTest, TestFreeTextSuggestions, ManagedSchemaRoundRobinCloudTest, TestSQLHandlerNonCloud, UpdateRequestProcessorFactoryTest, TestFastLRUCache, TestCoreAdminApis, TestHttpShardHandlerFactory, TestExclusionRuleCollectionAccess, ConfigSetsAPITest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, ShardRoutingTest, BasicZkTest, ClusterStateUpdateTest, LeaderElectionTest, TestZkChroot, ZkCLITest, TestDistributedGrouping, TestHashPartitioner, TestStressReorder, TestReload, TestCoreContainer, SimpleFacetsTest, SpellCheckComponentTest, PeerSyncTest, TestLazyCores, DistributedQueryElevationComponentTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, SolrRequestParserTest, SuggesterTest, WordBreakSolrSpellCheckerTest, TestUpdate, DirectUpdateHandlerOptimizeTest, TestRemoteStreaming, DistanceFunctionTest, DebugComponentTest, LukeRequestHandlerTest, TestQueryTypes, TermVectorComponentTest, RequiredFieldsTest, TestSolrQueryParser, SolrPluginUtilsTest, TestCollationField, MBeansHandlerTest, TestAnalyzedSuggestions, AlternateDirectoryTest, TestQuerySenderNoQuery, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, TestBinaryField, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionReloadTest, CollectionsAPIAsyncDistributedZkTest, DistribCursorPagingTest, DistributedQueueTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest]
   [junit4] Completed [523/729 (3!)] on J1 in 253.66s, 1 test, 1 failure <<< FAILURES!

[...truncated 44061 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 37 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/37/

2 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test

Error Message:
Mismatch in counts between replicas

Stack Trace:
java.lang.AssertionError: Mismatch in counts between replicas
        at __randomizedtesting.SeedInfo.seed([574A0B0E76428B14:DF1E34D4D8BEE6EC]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.RecoveryZkTest.assertShardConsistency(RecoveryZkTest.java:143)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:126)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([574A0B0E76428B14:DF1E34D4D8BEE6EC]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13347 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/init-core-data-001
   [junit4]   2> 4860828 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 4860832 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 4860833 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /hkse/wu
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4860906 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 4860918 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4860920 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4860930 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_37308_hdfs____k6l935/webapp
   [junit4]   2> 4861409 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37308
   [junit4]   2> 4861496 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4861497 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4861508 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_59592_datanode____.rffhr/webapp
   [junit4]   2> 4861977 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:59592
   [junit4]   2> 4862105 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4862106 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4862129 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60624_datanode____tav5yd/webapp
   [junit4]   2> 4862293 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:35660) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4862456 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x10edd65c3d3a3c: from storage DS-7cf99784-6d71-48a5-925a-db90f081c7df node DatanodeRegistration(127.0.0.1:53681, datanodeUuid=9acdf709-cfa9-4942-ae91-cffab6199131, infoPort=59557, infoSecurePort=0, ipcPort=37835, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4862456 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x10edd65c3d3a3c: from storage DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5 node DatanodeRegistration(127.0.0.1:53681, datanodeUuid=9acdf709-cfa9-4942-ae91-cffab6199131, infoPort=59557, infoSecurePort=0, ipcPort=37835, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4862719 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60624
   [junit4]   2> 4862853 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:35660) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4862862 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x10edd6745bd504: from storage DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585 node DatanodeRegistration(127.0.0.1:43710, datanodeUuid=ab5672b4-645f-446e-97b3-69f148aadf16, infoPort=50230, infoSecurePort=0, ipcPort=51109, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4862862 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x10edd6745bd504: from storage DS-febeccb2-bad1-408e-b719-39e14e3af424 node DatanodeRegistration(127.0.0.1:43710, datanodeUuid=ab5672b4-645f-446e-97b3-69f148aadf16, infoPort=50230, infoSecurePort=0, ipcPort=51109, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4863792 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4863792 INFO  (Thread-54437) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4863792 INFO  (Thread-54437) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4863793 ERROR (Thread-54437) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4863898 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkTestServer start zk server on port:56152
   [junit4]   2> 4863918 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4863919 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 4863920 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4863921 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4863921 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4863926 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4863930 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4863930 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4863931 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4863932 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4863932 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4863933 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4864024 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 4864025 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3c14ec1c{/hkse/wu,null,AVAILABLE}
   [junit4]   2> 4864025 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3b74f913{HTTP/1.1,[http/1.1]}{127.0.0.1:32804}
   [junit4]   2> 4864025 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server Started @4868762ms
   [junit4]   2> 4864025 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/hkse/wu, hostPort=32804, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores}
   [junit4]   2> 4864025 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4864025 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 4864026 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4864026 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4864026 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-04T08:19:38.363Z
   [junit4]   2> 4864034 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4864034 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/solr.xml
   [junit4]   2> 4864038 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4864041 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
   [junit4]   2> 4864090 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4864090 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32804_hkse%2Fwu
   [junit4]   2> 4864091 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.Overseer Overseer (id=98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) starting
   [junit4]   2> 4864096 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32804_hkse%2Fwu
   [junit4]   2> 4864097 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4864374 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4864383 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4864383 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4864384 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores
   [junit4]   2> 4864433 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4864434 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56152/solr ready
   [junit4]   2> 4864439 INFO  (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32804_hkse%252Fwu&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4864450 INFO  (OverseerThreadFactory-9664-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4864585 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4864585 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4864688 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4865608 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 4865650 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 4865714 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4865724 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 4865725 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4865725 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
   [junit4]   2> 4865725 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4865725 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4865725 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/]
   [junit4]   2> 4865726 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
   [junit4]   2> 4865733 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4865733 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4865742 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4865743 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data
   [junit4]   2> 4865760 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/index
   [junit4]   2> 4865765 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4865765 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4865768 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4865768 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
   [junit4]   2> 4865787 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5:NORMAL:127.0.0.1:53681|RBW], ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW]]} size 0
   [junit4]   2> 4865789 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741825_1001 size 72
   [junit4]   2> 4865799 WARN  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4865961 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4865961 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4865961 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4865981 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4865981 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4865982 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
   [junit4]   2> 4865999 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cc5391e[control_collection_shard1_replica_n1] main]
   [junit4]   2> 4866000 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4866000 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4866000 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 4866002 INFO  (searcherExecutor-9667-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6cc5391e[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4866002 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596412587147264
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4866010 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4866012 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4866114 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4866168 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4866169 INFO  (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1584
   [junit4]   2> 4866177 INFO  (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4866278 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4866458 INFO  (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4867178 INFO  (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32804_hkse%252Fwu&wt=javabin&version=2} status=0 QTime=2739
   [junit4]   2> 4867184 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4867185 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56152/solr ready
   [junit4]   2> 4867185 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4867186 INFO  (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4867187 INFO  (OverseerThreadFactory-9664-thread-2-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4867188 WARN  (OverseerThreadFactory-9664-thread-2-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4867394 INFO  (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4867402 INFO  (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=215
   [junit4]   2> 4867498 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001 of type NRT
   [junit4]   2> 4867498 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 4867499 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b8f407a{/hkse/wu,null,AVAILABLE}
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3ce27d3f{HTTP/1.1,[http/1.1]}{127.0.0.1:42830}
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server Started @4872237ms
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=42830, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores}
   [junit4]   2> 4867500 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4867500 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-04T08:19:41.837Z
   [junit4]   2> 4867527 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4867527 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/solr.xml
   [junit4]   2> 4867531 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4867534 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
   [junit4]   2> 4867543 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4867544 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4867545 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42830_hkse%2Fwu
   [junit4]   2> 4867546 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4867553 INFO  (zkCallback-26588-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4867579 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4867624 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4867634 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4867634 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4867636 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores
   [junit4]   2> 4867658 INFO  (qtp1753547614-161439) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42830_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4867659 INFO  (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4867660 INFO  (OverseerThreadFactory-9664-thread-3-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42830_hkse%2Fwu for creating new replica
   [junit4]   2> 4867661 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4867661 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4867764 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4868681 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 4868692 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
   [junit4]   2> 4868768 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4868798 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/]
   [junit4]   2> 4868800 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
   [junit4]   2> 4868824 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4868824 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4868845 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4868846 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data
   [junit4]   2> 4868875 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/index
   [junit4]   2> 4868884 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4868884 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4868896 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4868897 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
   [junit4]   2> 4868944 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-febeccb2-bad1-408e-b719-39e14e3af424:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
   [junit4]   2> 4868946 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-febeccb2-bad1-408e-b719-39e14e3af424:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
   [junit4]   2> 4868979 WARN  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4869017 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4869017 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4869017 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4869025 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4869026 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4869035 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
   [junit4]   2> 4869067 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@4552098b[collection1_shard2_replica_n0] main]
   [junit4]   2> 4869068 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4869068 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4869068 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 4869079 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596415813615616
   [junit4]   2> 4869081 INFO  (searcherExecutor-9678-thread-1-processing-n:127.0.0.1:42830_hkse%2Fwu x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@4552098b[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/ has no replicas
   [junit4]   2> 4869084 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4869086 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/ shard2
   [junit4]   2> 4869200 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4869247 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4869248 INFO  (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1587
   [junit4]   2> 4869250 INFO  (qtp1753547614-161439) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42830_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1592
   [junit4]   2> 4869350 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4869443 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001 of type NRT
   [junit4]   2> 4869443 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 4869472 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f722e82{/hkse/wu,null,AVAILABLE}
   [junit4]   2> 4869472 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7244f219{HTTP/1.1,[http/1.1]}{127.0.0.1:41450}
   [junit4]   2> 4869472 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server Started @4874209ms
   [junit4]   2> 4869472 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=41450, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores}
   [junit4]   2> 4869473 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4869473 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 4869473 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4869474 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4869474 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-04T08:19:43.811Z
   [junit4]   2> 4869476 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4869476 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/solr.xml
   [junit4]   2> 4869480 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4869498 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
   [junit4]   2> 4869524 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4869526 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4869527 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41450_hkse%2Fwu
   [junit4]   2> 4869528 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4869528 INFO  (zkCallback-26588-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4869538 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4869558 INFO  (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4869656 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4869660 INFO  (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4869667 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4869667 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4869672 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores
   [junit4]   2> 4869706 INFO  (qtp1753547614-161440) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41450_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4869715 INFO  (OverseerThreadFactory-9664-thread-4-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41450_hkse%2Fwu for creating new replica
   [junit4]   2> 4869717 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4869717 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4869821 INFO  (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4869821 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4870733 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 4870750 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema [collection1_shard1_replica_n0] Schema name=test
   [junit4]   2> 4871055 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4871066 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n0' using configuration from collection collection1, trusted=true
   [junit4]   2> 4871066 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n0' (registry 'solr.core.collection1.shard1.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
   [junit4]   2> 4871066 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
   [junit4]   2> 4871067 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4871067 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4871067 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [[collection1_shard1_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores/collection1_shard1_replica_n0], dataDir=[hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/]
   [junit4]   2> 4871068 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
   [junit4]   2> 4871074 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4871074 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4871081 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4871082 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data
   [junit4]   2> 4871107 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/index
   [junit4]   2> 4871112 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4871112 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4871118 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4871119 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
   [junit4]   2> 4871135 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|RBW], ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW]]} size 0
   [junit4]   2> 4871135 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
   [junit4]   2> 4871140 WARN  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4871186 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4871186 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4871186 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4871195 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4871195 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4871196 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
   [junit4]   2> 4871199 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@200818eb[collection1_shard1_replica_n0] main]
   [junit4]   2> 4871200 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4871201 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4871222 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 4871223 INFO  (searcherExecutor-9689-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu x:collection1_shard1_replica_n0 s:shard1 c:collection1) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [collection1_shard1_replica_n0] Registered new searcher Searcher@200818eb[collection1_shard1_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4871224 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596418062811136
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/ has no replicas
   [junit4]   2> 4871234 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4871237 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/ shard1
   [junit4]   2> 4871338 INFO  (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4871338 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4871388 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4871389 INFO  (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1  x:collection1_shard1_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1672
   [junit4]   2> 4871399 INFO  (qtp1753547614-161440) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41450_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1693
   [junit4]   2> 4871491 INFO  (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4871491 INFO  (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4871514 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001 of type NRT
   [junit4]   2> 4871514 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 4871515 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6227168e{/hkse/wu,null,AVAILABLE}
   [junit4]   2> 4871515 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2f960b78{HTTP/1.1,[http/1.1]}{127.0.0.1:43591}
   [junit4]   2> 4871515 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.e.j.s.Server Started @4876253ms
   [junit4]   2> 4871515 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=43591, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001/cores}
   [junit4]   2> 4871516 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4871525 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 4871525 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4871525 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4871525 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-04T08:19:45.862Z
   [junit4]   2> 4871535 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4871535 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001/solr.xml
   [junit4]   2> 4871541 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4871544 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
   [junit4]   2> 4871549 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 4871550 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4871552 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43591_hkse%2Fwu
   [junit4]   2> 4871553 INFO  (zkCallback-26588-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4871553 INFO  (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4871563 INFO  (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 4871563 INFO  (zkCallback-26593-thread-2-proc

[...truncated too long message...]

-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:34855) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1068949724-10.41.0.5-1504513889705 (Datanode Uuid 59b2686d-2d56-4bf1-8314-8f210305eb97) service to localhost/127.0.0.1:34855
   [junit4]   2> 5602691 WARN  (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 5602735 INFO  (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5602736 WARN  (296771641@qtp-118835101-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54539) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 5602737 WARN  (296771641@qtp-118835101-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54539) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 5602739 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:34855) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 5602739 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:34855) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1068949724-10.41.0.5-1504513889705 (Datanode Uuid 02c98771-33f5-4d2b-b478-76e0e77e02dd) service to localhost/127.0.0.1:34855
   [junit4]   2> 5602758 INFO  (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5602765 INFO  (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39453 39453
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001
   [junit4]   2> Sep 04, 2017 8:32:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=PostingsFormat(name=Memory), _version_=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), a_i=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1274, maxMBSortInHeap=5.470026215518962, sim=RandomSimilarity(queryNorm=false): {}, locale=hr-HR, timezone=Mexico/BajaNorte
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=199049240,total=522715136
   [junit4]   2> NOTE: All tests run in this JVM: [TestExclusionRuleCollectionAccess, CdcrVersionReplicationTest, AnalyticsMergeStrategyTest, HdfsBasicDistributedZkTest, TlogReplayBufferedWhileIndexingTest, SpellCheckComponentTest, DocValuesMultiTest, RestartWhileUpdatingTest, JsonLoaderTest, TestDFISimilarityFactory, TestBinaryResponseWriter, TestFiltering, TestCursorMarkWithoutUniqueKey, OpenCloseCoreStressTest, SolrJmxReporterCloudTest, BitVectorTest, DistributedQueryComponentCustomSortTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CollectionsAPIDistributedZkTest, TestFastLRUCache, NoCacheHeaderTest, TestMiniSolrCloudClusterSSL, RequestLoggingTest, TestHashQParserPlugin, TestClusterProperties, TestCoreAdminApis, SchemaVersionSpecificBehaviorTest, TestXIncludeConfig, CdcrReplicationDistributedZkTest, UpdateRequestProcessorFactoryTest, TestSerializedLuceneMatchVersion, CoreSorterTest, ZkStateReaderTest, TestCloudDeleteByQuery, DistributedMLTComponentTest, ActionThrottleTest, TestCoreContainer, TestSmileRequest, TestShardHandlerFactory, SpatialRPTFieldTypeTest, TestSolrXml, SOLR749Test, TestQueryTypes, TestValueSourceCache, TestNumericRangeQuery32, TestSweetSpotSimilarityFactory, DirectSolrConnectionTest, TestMacros, TestSolrCloudWithDelegationTokens, CacheHeaderTest, TestDocBasedVersionConstraints, BasicFunctionalityTest, AlternateDirectoryTest, JavabinLoaderTest, DateRangeFieldTest, HdfsCollectionsAPIDistributedZkTest, TestSolrFieldCacheBean, SecurityConfHandlerTest, ClassificationUpdateProcessorTest, TestTlogReplica, TestJsonFacetRefinement, TestTrieFacet, ResourceLoaderTest, InfoHandlerTest, TestDistributedGrouping, TestFieldCacheReopen, SaslZkACLProviderTest, SystemInfoHandlerTest, ShardRoutingTest, TestStressRecovery, DistanceFunctionTest, TestStressVersions, ConfigSetsAPITest, TestPayloadScoreQParserPlugin, TestFieldTypeCollectionResource, TestDynamicLoading, CollectionsAPIAsyncDistributedZkTest, AutoScalingHandlerTest, TestAuthorizationFramework, LoggingHandlerTest, TestUpdate, SpatialFilterTest, TestHttpShardHandlerFactory, TestPerFieldSimilarity, CheckHdfsIndexTest, TestSlowCompositeReaderWrapper, TestPostingsSolrHighlighter, TestCharFilters, DocValuesTest, TestSchemaResource, TestManagedResourceStorage, BlockDirectoryTest, FacetPivotSmallTest, TestSimpleTrackingShardHandler, DistribJoinFromCollectionTest, TestFieldSortValues, TestSQLHandlerNonCloud, MoveReplicaTest, BlockJoinFacetDistribTest, TestComplexPhraseLeadingWildcard, AddBlockUpdateTest, SoftAutoCommitTest, MetricUtilsTest, TestRTGBase, AsyncCallRequestStatusResponseTest, TestSegmentSorting, TestCloudPivotFacet, TestDynamicFieldResource, TestConfigSetsAPIExclusivity, TestAnalyzeInfixSuggestions, SolrCmdDistributorTest, ShufflingReplicaListTransformerTest, SampleTest, SolrCoreMetricManagerTest, DistributedFacetExistsSmallTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, SyncSliceTest, OverseerTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestHashPartitioner, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestJoin, TestReload, HardAutoCommitTest, DistributedTermsComponentTest, TestSolr4Spatial, SolrCoreTest, StatsComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTSTTest, TestStressLucene, TestCSVLoader, PolyFieldTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, LukeRequestHandlerTest, DisMaxRequestHandlerTest, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestOmitPositions, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, MBeansHandlerTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, BadComponentTest, TestSolrDeletionPolicy2, TestDocSet, NumericFieldsTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, EchoParamsTest, TestLMJelinekMercerSimilarityFactory, TimeZoneUtilsTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PreAnalyzedFieldTest, PrimUtilsTest, DOMUtilTest, TestSolrJ, TestUtils, ZkNodePropsTest, UUIDFieldTest, CircularListTest, DistributedIntervalFacetingTest, CdcrRequestHandlerTest, CollectionsAPISolrJTest, DeleteStatusTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerStatusTest, PeerSyncReplicationTest, RemoteQueryErrorTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework, TestCloudPseudoReturnFields, TestCloudRecovery, TestConfigSetsAPI, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLockTree, TestMiniSolrCloudCluster, TestOnReconnectListenerSupport, TestPullReplica, TestRandomFlRTGCloud, TestStressInPlaceUpdates, TestPolicyCloud, HdfsChaosMonkeyNothingIsSafeTest, HdfsRecoveryZkTest]
   [junit4] Completed [555/729 (2!)] on J1 in 41.48s, 1 test, 1 failure <<< FAILURES!

[...truncated 43900 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 38 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/38/

6 tests failed.
FAILED:  org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([A03631D71003B06D:27614C58815ACCED]:0)
        at org.apache.lucene.util.bkd.HeapPointWriter.writePackedValue(HeapPointWriter.java:107)
        at org.apache.lucene.util.bkd.HeapPointWriter.append(HeapPointWriter.java:128)
        at org.apache.lucene.util.bkd.PointReader.split(PointReader.java:68)
        at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1791)
        at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1008)
        at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
        at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.writeField(AssertingPointsFormat.java:137)
        at org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:183)
        at org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:203)
        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
        at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:399)
        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:502)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1717)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
        at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
        at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
        at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
        at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)


FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collMinRf_1x3_shard1_replica_n2",             "base_url":"http://127.0.0.1:39733/ggth/x",             "node_name":"127.0.0.1:39733_ggth%2Fx",             "state":"active",             "type":"NRT"},           "core_node2":{             "core":"collMinRf_1x3_shard1_replica_n1",             "base_url":"http://127.0.0.1:36295/ggth/x",             "node_name":"127.0.0.1:36295_ggth%2Fx",             "state":"active",             "type":"NRT"},           "core_node3":{             "core":"collMinRf_1x3_shard1_replica_n3",             "base_url":"http://127.0.0.1:46319/ggth/x",             "node_name":"127.0.0.1:46319_ggth%2Fx",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"3",     "tlogReplicas":"0"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"collection1_shard1_replica_n0",             "base_url":"http://127.0.0.1:39733/ggth/x",             "node_name":"127.0.0.1:39733_ggth%2Fx",             "state":"active",             "type":"NRT",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1_shard2_replica_n0",             "base_url":"http://127.0.0.1:39622/ggth/x",             "node_name":"127.0.0.1:39622_ggth%2Fx",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node3":{             "core":"collection1_shard2_replica_n1",             "base_url":"http://127.0.0.1:46319/ggth/x",             "node_name":"127.0.0.1:46319_ggth%2Fx",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:36295/ggth/x",             "node_name":"127.0.0.1:36295_ggth%2Fx",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"c8n_1x2_shard1_replica_n2",             "base_url":"http://127.0.0.1:39733/ggth/x",             "node_name":"127.0.0.1:39733_ggth%2Fx",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node2":{             "state":"down",             "base_url":"http://127.0.0.1:36295/ggth/x",             "core":"c8n_1x2_shard1_replica_n1",             "node_name":"127.0.0.1:36295_ggth%2Fx",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"2",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collMinRf_1x3_shard1_replica_n2",
            "base_url":"http://127.0.0.1:39733/ggth/x",
            "node_name":"127.0.0.1:39733_ggth%2Fx",
            "state":"active",
            "type":"NRT"},
          "core_node2":{
            "core":"collMinRf_1x3_shard1_replica_n1",
            "base_url":"http://127.0.0.1:36295/ggth/x",
            "node_name":"127.0.0.1:36295_ggth%2Fx",
            "state":"active",
            "type":"NRT"},
          "core_node3":{
            "core":"collMinRf_1x3_shard1_replica_n3",
            "base_url":"http://127.0.0.1:46319/ggth/x",
            "node_name":"127.0.0.1:46319_ggth%2Fx",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"collection1_shard1_replica_n0",
            "base_url":"http://127.0.0.1:39733/ggth/x",
            "node_name":"127.0.0.1:39733_ggth%2Fx",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1_shard2_replica_n0",
            "base_url":"http://127.0.0.1:39622/ggth/x",
            "node_name":"127.0.0.1:39622_ggth%2Fx",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "base_url":"http://127.0.0.1:46319/ggth/x",
            "node_name":"127.0.0.1:46319_ggth%2Fx",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:36295/ggth/x",
            "node_name":"127.0.0.1:36295_ggth%2Fx",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"c8n_1x2_shard1_replica_n2",
            "base_url":"http://127.0.0.1:39733/ggth/x",
            "node_name":"127.0.0.1:39733_ggth%2Fx",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node2":{
            "state":"down",
            "base_url":"http://127.0.0.1:36295/ggth/x",
            "core":"c8n_1x2_shard1_replica_n1",
            "node_name":"127.0.0.1:36295_ggth%2Fx",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"2",
    "tlogReplicas":"0"}}
        at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:4D3EA9CA327D5539]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
        at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:368)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv

Error Message:
java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error    request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2 Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f

Stack Trace:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error



request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
        at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:F37EF45616DC02D0]:0)
        at java.util.concurrent.FutureTask.report(FutureTask.java:122)
        at java.util.concurrent.FutureTask.get(FutureTask.java:192)
        at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:283)
        at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv(TestStressCloudBlindAtomicUpdates.java:195)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error



request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
        at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:411)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error



request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.doRandomAtomicUpdate(TestStressCloudBlindAtomicUpdates.java:370)
        at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:399)
        ... 6 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:4D3EA9CA327D5539]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 8728 lines...]
   [junit4] Suite: org.apache.lucene.search.TestInetAddressRangeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestInetAddressRangeQueries -Dtests.method=testRandomBig -Dtests.seed=A03631D71003B06D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=fr-CH -Dtests.timezone=Africa/Cairo -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   1104s J1 | TestInetAddressRangeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > at __randomizedtesting.SeedInfo.seed([A03631D71003B06D:27614C58815ACCED]:0)
   [junit4]    > at org.apache.lucene.util.bkd.HeapPointWriter.writePackedValue(HeapPointWriter.java:107)
   [junit4]    > at org.apache.lucene.util.bkd.HeapPointWriter.append(HeapPointWriter.java:128)
   [junit4]    > at org.apache.lucene.util.bkd.PointReader.split(PointReader.java:68)
   [junit4]    > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1791)
   [junit4]    > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1008)
   [junit4]    > at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
   [junit4]    > at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.writeField(AssertingPointsFormat.java:137)
   [junit4]    > at org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:183)
   [junit4]    > at org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:203)
   [junit4]    > at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
   [junit4]    > at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]    > at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]    > at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:399)
   [junit4]    > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:502)
   [junit4]    > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1717)
   [junit4]    > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
   [junit4]    > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
   [junit4]    > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
   [junit4]    > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
   [junit4]    > at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/J1/temp/lucene.search.TestInetAddressRangeQueries_A03631D71003B06D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Direct)}, docValues:{ipRangeField=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1019, maxMBSortInHeap=6.780535571636752, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=Africa/Cairo
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=252103960,total=371195904
   [junit4]   2> NOTE: All tests run in this JVM: [TestInetAddressRangeQueries]
   [junit4] Completed [12/13 (1!)] on J1 in 1160.35s, 4 tests, 1 error <<< FAILURES!

[...truncated 4 lines...]
   [junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/temp/junit4-J1-20170905_222219_749533904013510541752.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps/java_pid25144.hprof ...
   [junit4] Heap dump file created [592704075 bytes in 10.654 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 1561 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> Zá? 05, 2017 8:08:24 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1410)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:380)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    2) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=16701B8549ADDA26 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7183s J1 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
   [junit4]   2> Zá? 05, 2017 8:08:24 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Zá? 05, 2017 8:08:44 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$DualCrossingEdgeIterator.<init>(GeoComplexPolygon.java:908)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:326)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:39)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:375)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Zá? 05, 2017 8:08:44 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> Zá? 05, 2017 8:08:47 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:164)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:52)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:375)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Memory)}, docValues:{id=DocValuesFormat(name=Memory), point=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1595, maxMBSortInHeap=7.688325144528704, sim=RandomSimilarity(queryNorm=false): {}, locale=cs, timezone=America/Panama
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=144690264,total=532152320
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=16701B8549ADDA26 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
   [junit4] Completed [10/11 (1!)] on J1 in 7224.92s, 6 tests, 2 errors <<< FAILURES!

[...truncated 1694 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/init-core-data-001
   [junit4]   2> 84469 WARN  (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 84485 INFO  (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 84489 INFO  (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 84490 INFO  (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ggth/x
   [junit4]   2> 84681 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 84725 INFO  (Thread-32) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 84726 INFO  (Thread-32) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 84789 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.ZkTestServer start zk server on port:35247
   [junit4]   2> 84892 ERROR (Thread-32) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 85279 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 85281 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 85311 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 85312 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 85312 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 85313 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 85326 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 85327 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 85328 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 85329 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 85330 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 85411 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 88002 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 88022 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@199fcca0{/ggth/x,null,AVAILABLE}
   [junit4]   2> 88033 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6add72d3{HTTP/1.1,[http/1.1]}{127.0.0.1:37569}
   [junit4]   2> 88033 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.Server Started @100318ms
   [junit4]   2> 88034 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/tempDir-001/control/data, hostContext=/ggth/x, hostPort=36295, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores}
   [junit4]   2> 88034 ERROR (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 88034 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 88034 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 88034 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 88034 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-06T01:19:33.081Z
   [junit4]   2> 88051 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 88051 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/solr.xml
   [junit4]   2> 88120 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 88157 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35247/solr
   [junit4]   2> 88611 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 88612 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36295_ggth%2Fx
   [junit4]   2> 88629 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.Overseer Overseer (id=98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) starting
   [junit4]   2> 88927 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36295_ggth%2Fx
   [junit4]   2> 88929 INFO  (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 89748 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 89881 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 89943 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 89945 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores
   [junit4]   2> 90565 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 90565 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35247/solr ready
   [junit4]   2> 90610 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36930,localport=36295], receiveBufferSize:531000
   [junit4]   2> 90739 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51450], receiveBufferSize=530904
   [junit4]   2> 90810 INFO  (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36295_ggth%252Fx&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 90875 INFO  (OverseerThreadFactory-28-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 91178 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36934,localport=36295], receiveBufferSize:531000
   [junit4]   2> 91227 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51454], receiveBufferSize=530904
   [junit4]   2> 91232 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 91233 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 91419 INFO  (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 92529 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 93013 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 94663 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 94961 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 94962 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 94962 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 94962 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 95037 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=42.5146484375, floorSegmentMB=0.98046875, forceMergeDeletesPctAllowed=29.48434745238268, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19348825750878085
   [junit4]   2> 95198 WARN  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 95856 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 95872 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 95890 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 95904 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 95907 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6b848d07
   [junit4]   2> 95921 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b196792[control_collection_shard1_replica_n1] main]
   [junit4]   2> 95935 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 95935 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 95936 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 95983 INFO  (searcherExecutor-31-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1b196792[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 95983 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577751183129313280
   [junit4]   2> 96110 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 96111 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 96111 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/
   [junit4]   2> 96111 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 96111 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 96111 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 96126 INFO  (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 96142 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 96271 INFO  (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 96305 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 96369 INFO  (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5137
   [junit4]   2> 96386 INFO  (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 96482 INFO  (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 96904 INFO  (OverseerCollectionConfigSetProcessor-98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 97395 INFO  (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36295_ggth%252Fx&wt=javabin&version=2} status=0 QTime=6585
   [junit4]   2> 97633 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 97633 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35247/solr ready
   [junit4]   2> 97635 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 97650 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=37010,localport=36295], receiveBufferSize:531000
   [junit4]   2> 97749 INFO  (SocketProxy-Acceptor-36295) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51530], receiveBufferSize=530904
   [junit4]   2> 97751 INFO  (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 97769 INFO  (OverseerThreadFactory-28-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 97849 WARN  (OverseerThreadFactory-28-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 98290 INFO  (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 98290 INFO  (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=538
   [junit4]   2> 99063 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001 of type NRT
   [junit4]   2> 99081 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 99194 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fa1eb49{/ggth/x,null,AVAILABLE}
   [junit4]   2> 99195 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6359f9fb{HTTP/1.1,[http/1.1]}{127.0.0.1:34330}
   [junit4]   2> 99195 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.e.j.s.Server Started @111479ms
   [junit4]   2> 99195 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/ggth/x, hostPort=39622, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores}
   [junit4]   2> 99195 ERROR (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 99196 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 99196 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 99196 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 99196 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-06T01:19:44.243Z
   [junit4]   2> 99512 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 99512 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/solr.xml
   [junit4]   2> 99577 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 99641 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35247/solr
   [junit4]   2> 99889 INFO  (OverseerCollectionConfigSetProcessor-98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 100014 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 100016 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 100017 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39622_ggth%2Fx
   [junit4]   2> 100066 INFO  (zkCallback-27-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 100120 INFO  (zkCallback-39-thread-1-processing-n:127.0.0.1:39622_ggth%2Fx) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 100120 INFO  (zkCallback-34-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 101284 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 101545 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 101545 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 101565 INFO  (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores
   [junit4]   2> 102031 INFO  (SocketProxy-Acceptor-39622) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54794,localport=39622], receiveBufferSize:531000
   [junit4]   2> 102082 INFO  (SocketProxy-Acceptor-39622) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=34330,localport=38058], receiveBufferSize=530904
   [junit4]   2> 102119 INFO  (qtp1548631192-192) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39622_ggth%252Fx&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 102292 INFO  (OverseerThreadFactory-28-thread-3-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:39622_ggth%2Fx for creating new replica
   [junit4]   2> 102298 INFO  (SocketProxy-Acceptor-39622) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54826,localport=39622], receiveBufferSize:531000
   [junit4]   2> 102340 INFO  (SocketProxy-Acceptor-39622) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=34330,localport=38090], receiveBufferSize=530904
   [junit4]   2> 102357 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 102358 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 103750 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 104105 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
   [junit4]   2> 105917 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 106253 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
   [junit4]   2> 106253 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
   [junit4]   2> 106253 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 106253 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores/collection1_shard2_replica_n0/data/]
   [junit4]   2> 106300 INFO  (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2  x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=42.5146484375, floorSegmentMB=0.98046875, forceMergeDeletesPctAllowed=29.48434745238268, segmentsPerTier=46.0, maxCFS

[...truncated too long message...]

   2> 833443 INFO  (coreCloseExecutor-149-thread-1) [n:127.0.0.1:36393_solr c:test_col s:shard2 r:core_node3 x:test_col_shard2_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2df862d0 commitCommandVersion:0
   [junit4]   2> 833454 INFO  (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.test_col.shard3.leader, tag=1240110536
   [junit4]   2> 833454 INFO  (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 833454 INFO  (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@746f7d2a commitCommandVersion:0
   [junit4]   2> 833744 INFO  (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833744 INFO  (zkCallback-52-thread-9-processing-n:127.0.0.1:45766_solr) [n:127.0.0.1:45766_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833744 INFO  (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833744 INFO  (zkCallback-41-thread-9-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833755 WARN  (zkCallback-49-thread-9-processing-n:127.0.0.1:34849_solr) [n:127.0.0.1:34849_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 833788 INFO  (zkCallback-45-thread-9-processing-n:127.0.0.1:45746_solr) [n:127.0.0.1:45746_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833788 INFO  (zkCallback-47-thread-7-processing-n:127.0.0.1:36830_solr) [n:127.0.0.1:36830_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833788 INFO  (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
   [junit4]   2> 833869 INFO  (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
   [junit4]   2> 833871 INFO  (jetty-closer-2-thread-7) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4afadeca{/solr,null,UNAVAILABLE}
   [junit4]   2> 833922 INFO  (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
   [junit4]   2> 833922 INFO  (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
   [junit4]   2> 833922 INFO  (zkCallback-41-thread-8-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
   [junit4]   2> 833938 INFO  (zkCallback-52-thread-9-processing-n:127.0.0.1:45766_solr) [n:127.0.0.1:45766_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
   [junit4]   2> 833950 INFO  (jetty-closer-2-thread-10) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@b0cb78b{/solr,null,UNAVAILABLE}
   [junit4]   2> 834048 INFO  (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
   [junit4]   2> 834049 INFO  (jetty-closer-2-thread-5) [    ] o.a.s.c.Overseer Overseer (id=98609442762326046-127.0.0.1:42424_solr-n_0000000000) closing
   [junit4]   2> 834049 INFO  (OverseerStateUpdate-98609442762326046-127.0.0.1:42424_solr-n_0000000000) [n:127.0.0.1:42424_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:42424_solr
   [junit4]   2> 835358 WARN  (zkCallback-45-thread-9-processing-n:127.0.0.1:45746_solr) [n:127.0.0.1:45746_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835359 INFO  (jetty-closer-2-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@795ae91e{/solr,null,UNAVAILABLE}
   [junit4]   2> 835375 WARN  (zkCallback-47-thread-7-processing-n:127.0.0.1:36830_solr) [n:127.0.0.1:36830_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835375 INFO  (jetty-closer-2-thread-6) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@9edc191{/solr,null,UNAVAILABLE}
   [junit4]   2> 835423 WARN  (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835424 INFO  (jetty-closer-2-thread-9) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@191879b5{/solr,null,UNAVAILABLE}
   [junit4]   2> 835427 WARN  (zkCallback-41-thread-8-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835427 WARN  (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835428 INFO  (jetty-closer-2-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@31a2787b{/solr,null,UNAVAILABLE}
   [junit4]   2> 835440 INFO  (jetty-closer-2-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6cf8129{/solr,null,UNAVAILABLE}
   [junit4]   2> 835671 WARN  (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 835688 INFO  (jetty-closer-2-thread-5) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@71b537ee{/solr,null,UNAVAILABLE}
   [junit4]   2> 835750 ERROR (SUITE-TestStressCloudBlindAtomicUpdates-seed#[C56A96109C8138C1]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 835765 INFO  (SUITE-TestStressCloudBlindAtomicUpdates-seed#[C56A96109C8138C1]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44912 44912
   [junit4]   2> 836031 INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44912 44912
   [junit4]   2> 836187 WARN  (Thread-1) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 11 /solr/aliases.json
   [junit4]   2> 11 /solr/clusterprops.json
   [junit4]   2> 10 /solr/configs/org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates_config-set
   [junit4]   2> 10 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 4229 /solr/collections/test_col/state.json
   [junit4]   2> 11 /solr/clusterstate.json
   [junit4]   2> 2 /solr/overseer_elect/election/98609442762326046-127.0.0.1:42424_solr-n_0000000000
   [junit4]   2> 2 /solr/overseer_elect/election/98609442762326045-127.0.0.1:45766_solr-n_0000000003
   [junit4]   2> 2 /solr/overseer_elect/election/98609442762326033-127.0.0.1:37931_solr-n_0000000005
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 11 /solr/live_nodes
   [junit4]   2> 11 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressCloudBlindAtomicUpdates_C56A96109C8138C1-001
   [junit4]   2> Sep 06, 2017 1:32:01 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=SimpleText, sim=RandomSimilarity(queryNorm=false): {}, locale=sv-SE, timezone=Europe/Zaporozhye
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=210055568,total=405274624
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressCloudBlindAtomicUpdates]
   [junit4] Completed [25/729 (3!)] on J1 in 844.25s, 5 tests, 1 error <<< FAILURES!

[...truncated 10020 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:818: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid25144.hprof

Total time: 766 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 39 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/39/

3 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
.response.numFound:0!=1

Stack Trace:
junit.framework.AssertionFailedError: .response.numFound:0!=1
        at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
        at junit.framework.Assert.fail(Assert.java:50)
        at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
        at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:940)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:607)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
        at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkQueries(AbstractFullDistribZkTestBase.java:1050)
        at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:102)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collMinRf_1x3_shard1_replica_t3",             "base_url":"http://127.0.0.1:56032/m/o",             "node_name":"127.0.0.1:56032_m%2Fo",             "state":"active",             "type":"TLOG"},           "core_node2":{             "core":"collMinRf_1x3_shard1_replica_t1",             "base_url":"http://127.0.0.1:53584/m/o",             "node_name":"127.0.0.1:53584_m%2Fo",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node3":{             "core":"collMinRf_1x3_shard1_replica_t2",             "base_url":"http://127.0.0.1:38128/m/o",             "node_name":"127.0.0.1:38128_m%2Fo",             "state":"active",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"0",     "tlogReplicas":"3"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"collection1_shard1_replica_t0",             "base_url":"http://127.0.0.1:53584/m/o",             "node_name":"127.0.0.1:53584_m%2Fo",             "state":"active",             "type":"TLOG",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1_shard2_replica_t0",             "base_url":"http://127.0.0.1:56032/m/o",             "node_name":"127.0.0.1:56032_m%2Fo",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node3":{             "core":"collection1_shard2_replica_t1",             "base_url":"http://127.0.0.1:41484/m/o",             "node_name":"127.0.0.1:41484_m%2Fo",             "state":"active",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:38128/m/o",             "node_name":"127.0.0.1:38128_m%2Fo",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"c8n_1x2_shard1_replica_t1",             "base_url":"http://127.0.0.1:56032/m/o",             "node_name":"127.0.0.1:56032_m%2Fo",             "state":"active",             "type":"TLOG",             "leader":"true"},           "core_node2":{             "state":"down",             "base_url":"http://127.0.0.1:38128/m/o",             "core":"c8n_1x2_shard1_replica_t2",             "node_name":"127.0.0.1:38128_m%2Fo",             "type":"TLOG"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"0",     "tlogReplicas":"2"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collMinRf_1x3_shard1_replica_t3",
            "base_url":"http://127.0.0.1:56032/m/o",
            "node_name":"127.0.0.1:56032_m%2Fo",
            "state":"active",
            "type":"TLOG"},
          "core_node2":{
            "core":"collMinRf_1x3_shard1_replica_t1",
            "base_url":"http://127.0.0.1:53584/m/o",
            "node_name":"127.0.0.1:53584_m%2Fo",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node3":{
            "core":"collMinRf_1x3_shard1_replica_t2",
            "base_url":"http://127.0.0.1:38128/m/o",
            "node_name":"127.0.0.1:38128_m%2Fo",
            "state":"active",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"0",
    "tlogReplicas":"3"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"collection1_shard1_replica_t0",
            "base_url":"http://127.0.0.1:53584/m/o",
            "node_name":"127.0.0.1:53584_m%2Fo",
            "state":"active",
            "type":"TLOG",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1_shard2_replica_t0",
            "base_url":"http://127.0.0.1:56032/m/o",
            "node_name":"127.0.0.1:56032_m%2Fo",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "base_url":"http://127.0.0.1:41484/m/o",
            "node_name":"127.0.0.1:41484_m%2Fo",
            "state":"active",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:38128/m/o",
            "node_name":"127.0.0.1:38128_m%2Fo",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"c8n_1x2_shard1_replica_t1",
            "base_url":"http://127.0.0.1:56032/m/o",
            "node_name":"127.0.0.1:56032_m%2Fo",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node2":{
            "state":"down",
            "base_url":"http://127.0.0.1:38128/m/o",
            "core":"c8n_1x2_shard1_replica_t2",
            "node_name":"127.0.0.1:38128_m%2Fo",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"0",
    "tlogReplicas":"2"}}
        at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
        at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:368)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11658 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/init-core-data-001
   [junit4]   2> 658351 WARN  (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 658352 INFO  (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 658363 INFO  (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 658363 INFO  (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /m/o
   [junit4]   2> 658371 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 658379 INFO  (Thread-628) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 658379 INFO  (Thread-628) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 658477 ERROR (Thread-628) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 658747 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkTestServer start zk server on port:52997
   [junit4]   2> 658838 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 658839 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 658841 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 658842 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 658843 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 658843 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 658844 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 658845 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 658845 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 658846 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 658847 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 658848 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 659388 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 659389 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c236413{/m/o,null,AVAILABLE}
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@752de79e{HTTP/1.1,[http/1.1]}{127.0.0.1:49023}
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server Started @667954ms
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/control/data, replicaType=NRT, hostContext=/m/o, hostPort=38128, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores}
   [junit4]   2> 659390 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 659390 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-07T07:05:47.555Z
   [junit4]   2> 659396 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 659396 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/solr.xml
   [junit4]   2> 659414 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 659436 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
   [junit4]   2> 659756 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 659757 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38128_m%2Fo
   [junit4]   2> 659757 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.Overseer Overseer (id=98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) starting
   [junit4]   2> 659813 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38128_m%2Fo
   [junit4]   2> 659829 INFO  (OverseerStateUpdate-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 659931 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 659959 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 659959 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 659961 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores
   [junit4]   2> 660028 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 660028 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52997/solr ready
   [junit4]   2> 660029 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57347,localport=38128], receiveBufferSize:531000
   [junit4]   2> 660122 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60031], receiveBufferSize=530904
   [junit4]   2> 660130 INFO  (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38128_m%252Fo&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 660147 INFO  (OverseerThreadFactory-1446-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 660272 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57351,localport=38128], receiveBufferSize:531000
   [junit4]   2> 660281 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60035], receiveBufferSize=530904
   [junit4]   2> 660289 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 660289 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 660393 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 661364 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 661465 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 661818 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 661834 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 661834 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 661834 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 661834 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 661848 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
   [junit4]   2> 661850 WARN  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 661935 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 661935 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 661970 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 661970 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 661972 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 661972 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@242e154[control_collection_shard1_replica_n1] main]
   [junit4]   2> 661973 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 661973 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 661974 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 661975 INFO  (searcherExecutor-1449-thread-1-processing-n:127.0.0.1:38128_m%2Fo x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@242e154[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 661975 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863558086000640
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 661988 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 661991 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 662185 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 662233 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 662237 INFO  (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1948
   [junit4]   2> 662413 INFO  (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 662513 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 663413 INFO  (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38128_m%252Fo&wt=javabin&version=2} status=0 QTime=3283
   [junit4]   2> 663499 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 663500 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52997/solr ready
   [junit4]   2> 663500 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 663501 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57361,localport=38128], receiveBufferSize:531000
   [junit4]   2> 663512 INFO  (SocketProxy-Acceptor-38128) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60045], receiveBufferSize=530904
   [junit4]   2> 663520 INFO  (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 663539 INFO  (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 663540 INFO  (OverseerThreadFactory-1446-thread-2-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 663540 WARN  (OverseerThreadFactory-1446-thread-2-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 663927 INFO  (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 663927 INFO  (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=407
   [junit4]   2> 664323 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001 of type TLOG
   [junit4]   2> 664324 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 664411 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23303f8c{/m/o,null,AVAILABLE}
   [junit4]   2> 664411 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@663eb2d5{HTTP/1.1,[http/1.1]}{127.0.0.1:57009}
   [junit4]   2> 664411 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server Started @672975ms
   [junit4]   2> 664411 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=56032, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores}
   [junit4]   2> 664412 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 664412 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 664412 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 664412 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 664412 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-07T07:05:52.577Z
   [junit4]   2> 664439 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 664439 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/solr.xml
   [junit4]   2> 664444 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 664464 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
   [junit4]   2> 664983 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 664984 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 664986 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56032_m%2Fo
   [junit4]   2> 664987 INFO  (zkCallback-720-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 664987 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 664992 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 665233 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 665244 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 665245 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 665246 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores
   [junit4]   2> 665312 INFO  (SocketProxy-Acceptor-56032) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54042,localport=56032], receiveBufferSize:531000
   [junit4]   2> 665332 INFO  (SocketProxy-Acceptor-56032) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=57009,localport=42862], receiveBufferSize=530904
   [junit4]   2> 665337 INFO  (qtp1554326178-3624) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56032_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 665352 INFO  (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 665353 INFO  (OverseerThreadFactory-1446-thread-3-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56032_m%2Fo for creating new replica
   [junit4]   2> 665356 INFO  (SocketProxy-Acceptor-56032) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54045,localport=56032], receiveBufferSize:531000
   [junit4]   2> 665370 INFO  (SocketProxy-Acceptor-56032) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=57009,localport=42865], receiveBufferSize=530904
   [junit4]   2> 665375 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 665377 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 665482 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 666411 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 666468 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema [collection1_shard2_replica_t0] Schema name=test
   [junit4]   2> 666768 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 666781 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t0' using configuration from collection collection1, trusted=true
   [junit4]   2> 666787 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t0' (registry 'solr.core.collection1.shard2.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 666787 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 666787 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [[collection1_shard2_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores/collection1_shard2_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores/collection1_shard2_replica_t0/data/]
   [junit4]   2> 666793 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
   [junit4]   2> 666798 WARN  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 666839 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 666840 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 666841 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 666841 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 666843 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 666844 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@3804fd18[collection1_shard2_replica_t0] main]
   [junit4]   2> 666845 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 666848 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 666849 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 666849 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863563196760064
   [junit4]   2> 666851 INFO  (searcherExecutor-1460-thread-1-processing-n:127.0.0.1:56032_m%2Fo x:collection1_shard2_replica_t0 s:shard2 c:collection1) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [collection1_shard2_replica_t0] Registered new searcher Searcher@3804fd18[collection1_shard2_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/ has no replicas
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 666858 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ZkController collection1_shard2_replica_t0 stopping background replication from leader
   [junit4]   2> 666861 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/ shard2
   [junit4]   2> 666962 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 667012 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 667014 INFO  (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2  x:collection1_shard2_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1638
   [junit4]   2> 667018 INFO  (qtp1554326178-3624) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56032_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1680
   [junit4]   2> 667158 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 667360 INFO  (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 667365 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001 of type TLOG
   [junit4]   2> 667366 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 667443 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7046306e{/m/o,null,AVAILABLE}
   [junit4]   2> 667444 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@18c9c2ea{HTTP/1.1,[http/1.1]}{127.0.0.1:55541}
   [junit4]   2> 667444 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server Started @676008ms
   [junit4]   2> 667444 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=53584, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores}
   [junit4]   2> 667444 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 667451 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 667451 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 667451 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 667451 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-07T07:05:55.616Z
   [junit4]   2> 667510 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 667510 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/solr.xml
   [junit4]   2> 667519 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 667538 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
   [junit4]   2> 667749 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 667750 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 667752 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53584_m%2Fo
   [junit4]   2> 667753 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 667753 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 667753 INFO  (zkCallback-720-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 667771 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 667881 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 667899 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 667899 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 667900 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores
   [junit4]   2> 667934 INFO  (SocketProxy-Acceptor-53584) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34819,localport=53584], receiveBufferSize:531000
   [junit4]   2> 667938 INFO  (SocketProxy-Acceptor-53584) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=55541,localport=37275], receiveBufferSize=530904
   [junit4]   2> 667942 INFO  (qtp704117609-3658) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:53584_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 667961 INFO  (OverseerThreadFactory-1446-thread-4-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:53584_m%2Fo for creating new replica
   [junit4]   2> 667962 INFO  (SocketProxy-Acceptor-53584) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34822,localport=53584], receiveBufferSize:531000
   [junit4]   2> 667966 INFO  (SocketProxy-Acceptor-53584) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=55541,localport=37278], receiveBufferSize=530904
   [junit4]   2> 667966 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 667966 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 668072 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 668072 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 668985 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 669009 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema [collection1_shard1_replica_t0] Schema name=test
   [junit4]   2> 669391 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 669425 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t0' using configuration from collection collection1, trusted=true
   [junit4]   2> 669425 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t0' (registry 'solr.core.collection1.shard1.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 669425 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 669425 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [[collection1_shard1_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores/collection1_shard1_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores/collection1_shard1_replica_t0/data/]
   [junit4]   2> 669440 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
   [junit4]   2> 669448 WARN  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 669540 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 669540 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 669541 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 669541 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 669542 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 669542 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d6f0057[collection1_shard1_replica_t0] main]
   [junit4]   2> 669543 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 669543 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 669544 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 669544 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863566022672384
   [junit4]   2> 669550 INFO  (searcherExecutor-1471-thread-1-processing-n:127.0.0.1:53584_m%2Fo x:collection1_shard1_replica_t0 s:shard1 c:collection1) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [collection1_shard1_replica_t0] Registered new searcher Searcher@6d6f0057[collection1_shard1_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 669552 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 669552 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 669552 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/
   [junit4]   2> 669832 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670062 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 669832 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670063 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/ has no replicas
   [junit4]   2> 670063 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 670063 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ZkController collection1_shard1_replica_t0 stopping background replication from leader
   [junit4]   2> 670339 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/ shard1
   [junit4]   2> 670441 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670441 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670490 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 670492 INFO  (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1  x:collection1_shard1_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2526
   [junit4]   2> 670494 INFO  (qtp704117609-3658) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:53584_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2551
   [junit4]   2> 670711 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670711 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 670934 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001 of type TLOG
   [junit4]   2> 670935 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 670941 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@642ac4f4{/m/o,null,AVAILABLE}
   [junit4]   2> 670941 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@556207f0{HTTP/1.1,[http/1.1]}{127.0.0.1:39720}
   [junit4]   2> 670941 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.e.j.s.Server Started @679505ms
   [junit4]   2> 670941 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=41484, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/cores}
   [junit4]   2> 670942 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 670943 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 670943 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 670943 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 670943 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-07T07:05:59.108Z
   [junit4]   2> 670976 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 670976 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/solr.xml
   [junit4]   2> 671022 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 671027 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
   [junit4]   2> 671316 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 671317 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 671319 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41484_m%2Fo
   [junit4]   2> 671330 INFO  (zkCallback-737-thread-1-processing-n:127.0.0.1:41484_m%2Fo) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 671347 INFO  (zkCallback-720-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 671347 INFO  (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 671348 INFO  (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 671348 INFO  (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 671442 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 671459 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 671459 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
   [junit4]   2> 671464 INFO  (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/cores
   [junit4]   2> 671554 INFO  (qtp704117609-3659) [n:127.0.0.1:53584_m%2Fo    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41484_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 671572 INFO  (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 671727 INFO  (OverseerThreadFactory-1446-thread-5-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41484_m%2Fo for creating new replica
   [junit4]   2> 671728 INFO  (SocketProxy-Acceptor-41484) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33479,localport=41484], receiveBufferSize:531000
   [junit4]   2> 671751 INFO  (SocketProxy-Acceptor-41484) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39720,localport=37358], receiveBufferSize=530904
   [junit4]   2> 671758 INFO  (qtp706179858-3692) [n:127.0.0.1:41484_m%2Fo    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t1&action=CREATE&col

[...truncated too long message...]

er IncrementalBlockReportManager interrupted
   [junit4]   2> 5910593 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:59049) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-761140916-10.41.0.5-1504773134652 (Datanode Uuid 44d47a55-e14e-402f-b52c-e6c5b61dfb45) service to localhost/127.0.0.1:59049
   [junit4]   2> 5910602 WARN  (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 5910635 INFO  (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5910635 WARN  (1440079205@qtp-938187338-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54992) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 5910636 WARN  (1440079205@qtp-938187338-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54992) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 5910740 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:59049) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 5910740 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:59049) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-761140916-10.41.0.5-1504773134652 (Datanode Uuid 096f87eb-da2f-4576-b538-7114bfdff4f9) service to localhost/127.0.0.1:59049
   [junit4]   2> 5910785 INFO  (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5910785 WARN  (68133024@qtp-1610702844-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34307) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 5910785 WARN  (68133024@qtp-1610702844-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34307) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001
   [junit4]   2> Sep 07, 2017 8:33:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=BlockTreeOrds(blocksize=128), SubjectTerms_mfacet=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), oddField_s=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=Memory), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo_b=PostingsFormat(name=LuceneVarGapFixedInterval), id=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{other_tl1=DocValuesFormat(name=Memory), range_facet_l_dv=DocValuesFormat(name=Asserting), n_l1=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Memory), n_dt1=DocValuesFormat(name=Lucene70), a_i1=DocValuesFormat(name=Memory), n_td1=DocValuesFormat(name=Lucene70), n_d1=DocValuesFormat(name=Memory), range_facet_l=DocValuesFormat(name=Memory), n_f1=DocValuesFormat(name=Lucene70), n_ti1=DocValuesFormat(name=Asserting), n_tl1=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), n_tf1=DocValuesFormat(name=Memory), n_tdt1=DocValuesFormat(name=Memory), id_i1=DocValuesFormat(name=Lucene70), foo_d=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Memory), intDvoDefault=DocValuesFormat(name=Direct), foo_f=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1413, maxMBSortInHeap=5.826986203817525, sim=RandomSimilarity(queryNorm=true): {}, locale=ga-IE, timezone=Asia/Dili
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=152052264,total=504365056
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedFacetExistsSmallTest, TestSolrCLIRunExample, DistributedIntervalFacetingTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CdcrVersionReplicationTest, OpenCloseCoreStressTest, DistribJoinFromCollectionTest, CheckHdfsIndexTest, SampleTest, SolrCoreMetricManagerTest, TlogReplayBufferedWhileIndexingTest, TestSolrQueryResponse, TestPostingsSolrHighlighter, AddBlockUpdateTest, TestUpdate, TestCoreContainer, StatelessScriptUpdateProcessorFactoryTest, TestDynamicFieldResource, TestDFISimilarityFactory, SystemInfoHandlerTest, DistributedMLTComponentTest, ZkStateReaderTest, TestMaxScoreQueryParser, SaslZkACLProviderTest, TestInitParams, DocValuesNotIndexedTest, AtomicUpdatesTest, BlockJoinFacetDistribTest, TestStressVersions, TestSubQueryTransformerCrossCore, TestSerializedLuceneMatchVersion, MissingSegmentRecoveryTest, MoreLikeThisHandlerTest, CustomCollectionTest, TestJsonFacetRefinement, TestScoreJoinQPNoScore, TestPullReplica, LukeRequestHandlerTest, TestStressRecovery, SpellCheckComponentTest, TestHashQParserPlugin, TestRandomRequestDistribution, TestManagedSynonymFilterFactory, NoCacheHeaderTest, RemoteQueryErrorTest, FacetPivotSmallTest, SpatialRPTFieldTypeTest, DistributedQueryComponentCustomSortTest, SolrCloudExampleTest, ClassificationUpdateProcessorTest, TestLRUCache, TestLeaderElectionZkExpiry, TestFastLRUCache, HighlighterTest, HdfsCollectionsAPIDistributedZkTest, MetricUtilsTest, TestOrdValues, CloudMLTQParserTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestRTGBase, TestSha256AuthenticationProvider, TestSolrCloudWithHadoopAuthPlugin, TestSimpleTrackingShardHandler, SchemaVersionSpecificBehaviorTest, DataDrivenBlockJoinTest, SortSpecParsingTest, ConvertedLegacyTest, TestLockTree, LoggingHandlerTest, TestPerFieldSimilarity, TestNumericRangeQuery32, TestFieldSortValues, FileUtilsTest, BasicFunctionalityTest, TestExclusionRuleCollectionAccess, ShufflingReplicaListTransformerTest, FileBasedSpellCheckerTest, TestMaxTokenLenTokenizer, DirectSolrConnectionTest, AlternateDirectoryTest, SolrJmxReporterCloudTest, TestXIncludeConfig, RuleEngineTest, TestReloadDeadlock, ManagedSchemaRoundRobinCloudTest, SolrCloudReportersTest, TestManagedResourceStorage, CollectionsAPIDistributedZkTest, TestSolrCloudWithDelegationTokens, TestValueSourceCache, JavabinLoaderTest, TestSearchPerf, CollectionsAPIAsyncDistributedZkTest, TestSmileRequest, TestCloudPivotFacet, TestClassNameShortening, HdfsBasicDistributedZkTest, TestStressCloudBlindAtomicUpdates, AsyncCallRequestStatusResponseTest, SliceStateTest, HdfsWriteToMultipleCollectionsTest, DistributedFacetPivotSmallAdvancedTest, ActionThrottleTest, AnalyticsMergeStrategyTest, TestSQLHandlerNonCloud, CdcrReplicationDistributedZkTest, TestClusterProperties, CSVRequestHandlerTest, IndexSchemaRuntimeFieldTest, TestAuthorizationFramework, TestCharFilters, AutoScalingHandlerTest, HLLSerializationTest, TestConfigOverlay, TestLegacyNumericUtils, BooleanFieldTest, TestFieldTypeCollectionResource, TestQueryTypes, ScriptEngineTest, TestHalfAndHalfDocValues, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, ZkCLITest, TestDistributedSearch, ChaosMonkeySafeLeaderWithPullReplicasTest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, MoveReplicaHDFSTest, MoveReplicaTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerTaskQueueTest, ReplicationFactorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverUtilsTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestMiniSolrCloudClusterSSL, TestRandomFlRTGCloud, TestRequestForwarding, TestSegmentSorting, TestStressInPlaceUpdates, TestTlogReplica, TestTolerantUpdateProcessorRandomCloud, HdfsBasicDistributedZk2Test]
   [junit4] Completed [560/729 (3!)] on J1 in 69.81s, 1 test, 1 failure <<< FAILURES!

[...truncated 43843 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 40 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/40/

8 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestBackwardsCompatibility

Error Message:
Index name 6.6.1-cfs not found

Stack Trace:
java.lang.AssertionError: Index name 6.6.1-cfs not found
        at __randomizedtesting.SeedInfo.seed([277DA2692EEAC86F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.lucene.index.TestBackwardsCompatibility.beforeClass(TestBackwardsCompatibility.java:470)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:898863BA00DF759F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:222)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
11 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:     1) Thread[id=18356, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    2) Thread[id=18151, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    3) Thread[id=18149, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:90)    5) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    6) Thread[id=18358, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    7) Thread[id=18359, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)         at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)         at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)    8) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)         at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)    9) Thread[id=18472, name=StoppableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   10) Thread[id=18361, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=18357, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 11 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
   1) Thread[id=18356, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   2) Thread[id=18151, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   3) Thread[id=18149, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:90)
   5) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   6) Thread[id=18358, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   7) Thread[id=18359, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
   8) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
        at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)
   9) Thread[id=18472, name=StoppableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  10) Thread[id=18361, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=18357, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
        at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=18358, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)    2) Thread[id=18359, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)         at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)         at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)         at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)         at java.security.AccessController.doPrivileged(Native Method)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)         at java.io.PrintStream.flush(PrintStream.java:338)         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at java.io.PrintStream.write(PrintStream.java:482)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)         at java.io.PrintStream.write(PrintStream.java:527)         at java.io.PrintStream.print(PrintStream.java:669)         at java.io.PrintStream.println(PrintStream.java:823)         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)         at java.lang.Throwable.printEnclosedStackTrace(Throwable.java:697)         at java.lang.Throwable.printStackTrace(Throwable.java:667)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57)    3) Thread[id=18356, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)         at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)         at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)         at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    4) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    5) Thread[id=18357, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.io.PrintStream.println(PrintStream.java:805)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)    6) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.util.WeakHashMap.getTable(WeakHashMap.java:350)         at java.util.WeakHashMap.getEntry(WeakHashMap.java:427)         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)         at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)    7) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=18358, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
   2) Thread[id=18359, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
        at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
        at java.security.AccessController.doPrivileged(Native Method)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
        at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.write(PrintStream.java:482)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
        at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
        at java.io.PrintStream.write(PrintStream.java:527)
        at java.io.PrintStream.print(PrintStream.java:669)
        at java.io.PrintStream.println(PrintStream.java:823)
        at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
        at java.lang.Throwable.printEnclosedStackTrace(Throwable.java:697)
        at java.lang.Throwable.printStackTrace(Throwable.java:667)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57)
   3) Thread[id=18356, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
        at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   4) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   5) Thread[id=18357, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.PrintStream.println(PrintStream.java:805)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
   6) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
        at java.util.WeakHashMap.getEntry(WeakHashMap.java:427)
        at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
        at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)
        at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
        at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
        at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)
   7) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest

Error Message:
Error from server at http://127.0.0.1:47708/solr: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:47708/solr: ADDREPLICA failed to create replica
        at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:6FEE4FAF9B9A925]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest(HdfsCollectionsAPIDistributedZkTest.java:127)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}]> but was:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}, {indexVersion=1504863284898,generation=3,filelist=[_1fp.cfe, _1fp.cfs, _1fp.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}]> but was:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}, {indexVersion=1504863284898,generation=3,filelist=[_1fp.cfe, _1fp.cfs, _1fp.si, segments_3]}]>
        at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:240B4750DE6B1664]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.logging.TestLogWatcher.testLog4jWatcher

Error Message:
expected:<3> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<1>
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.logging.TestLogWatcher.testLog4jWatcher(TestLogWatcher.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java