SSL question

classic Classic list List threaded Threaded
2 messages Options
Reply | Threaded
Open this post in threaded view
|

SSL question

Tony Sargent
Hello,

I have set up a SolrCloud with 3 Solr nodes (v7.7) and 3 external Zookeepers (3.4.14). However, whenever I enable SSL I get these errors in the logs that for some reason Solr (or maybe ZK) still wants to communicate via http. I am able to pull up admin Console via https but when I click on Cloud/Nodes to look at metrics that’s when I see this message in the logs.
In the documentation it’s showing to pass “-name urlScheme -val https”  to Zookeeper also but external zkServer.sh doesn’t have that option.
What am I doing wrong? I am pulling my hair out, trying to get this going.
JKS has Key and a SAN certificate that includes all of the node names. JKS is loaded on all Solr nodes.
This is running inside of docker contains and I have passed the following variables:

      - SOLR_MODE=solrcloud
      - ZK_HOST=mynode1_fqdn:2181,mynode2_fqdn:2181,mynode2_fqdn:2181
      - SOLR_SSL_ENABLED=true
      - SOLR_SSL_KEY_STORE=/opt/solr/etc/solr-ssl.keystore.jks
      - SOLR_SSL_KEY_STORE_PASSWORD=secret
      - SOLR_SSL_TRUST_STORE=/opt/solr/etc/solr-ssl.keystore.jks
      - SOLR_SSL_TRUST_STORE_PASSWORD=secret
      - SOLR_SSL_NEED_CLIENT_AUTH=false
      - SOLR_SSL_WANT_CLIENT_AUTH=true
      - SOLR_SSL_CHECK_PEER_NAME=false
      - SOLR_URL_SCHEME=https





2019-05-16 22:15:44.999 WARN  (qtp841262455-21) [   ] o.a.s.h.a.AdminHandlersProxy Exception when fetching result from node mynode1_fqdn:8983_solr
java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://mynode1_fqdn:8983/solr
                at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:?]
                at java.util.concurrent.FutureTask.get(FutureTask.java:205) ~[?:?]
                at org.apache.solr.handler.admin.AdminHandlersProxy.maybeProxyToNodes(AdminHandlersProxy.java:102) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:136) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.handler.admin.InfoHandler.handle(InfoHandler.java:91) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:81) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
                at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1602) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) [jetty-security-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335) [jetty-rewrite-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.Server.handle(Server.java:502) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
                at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://mynode1_fqdn:8983/solr
                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
                at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
                at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
                ... 1 more
Caused by: org.apache.http.client.ClientProtocolException
                at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:187) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
                at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
                at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
                at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
                ... 1 more
Caused by: org.apache.http.ProtocolException: The server failed to respond with a valid HTTP response
                at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:149) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) ~[httpcore-4.4.10.jar:4.4.10]
                at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) ~[httpcore-4.4.10.jar:4.4.10]
                at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) ~[httpcore-4.4.10.jar:4.4.10]
                at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) ~[httpcore-4.4.10.jar:4.4.10]
                at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.6.jar:4.5.6]
                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]



Tony Sargent
Unix Middle Tier Admin, Web Operations
801.442.5579 | [hidden email]<mailto:[hidden email]>
Reply | Threaded
Open this post in threaded view
|

Re: SSL question

Jörn Franke
You have to do a zookeeper setting:
server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clusterprop -name urlScheme -val https

> Am 17.05.2019 um 00:36 schrieb Tony Sargent <[hidden email]>:
>
> Hello,
>
> I have set up a SolrCloud with 3 Solr nodes (v7.7) and 3 external Zookeepers (3.4.14). However, whenever I enable SSL I get these errors in the logs that for some reason Solr (or maybe ZK) still wants to communicate via http. I am able to pull up admin Console via https but when I click on Cloud/Nodes to look at metrics that’s when I see this message in the logs.
> In the documentation it’s showing to pass “-name urlScheme -val https”  to Zookeeper also but external zkServer.sh doesn’t have that option.
> What am I doing wrong? I am pulling my hair out, trying to get this going.
> JKS has Key and a SAN certificate that includes all of the node names. JKS is loaded on all Solr nodes.
> This is running inside of docker contains and I have passed the following variables:
>
>      - SOLR_MODE=solrcloud
>      - ZK_HOST=mynode1_fqdn:2181,mynode2_fqdn:2181,mynode2_fqdn:2181
>      - SOLR_SSL_ENABLED=true
>      - SOLR_SSL_KEY_STORE=/opt/solr/etc/solr-ssl.keystore.jks
>      - SOLR_SSL_KEY_STORE_PASSWORD=secret
>      - SOLR_SSL_TRUST_STORE=/opt/solr/etc/solr-ssl.keystore.jks
>      - SOLR_SSL_TRUST_STORE_PASSWORD=secret
>      - SOLR_SSL_NEED_CLIENT_AUTH=false
>      - SOLR_SSL_WANT_CLIENT_AUTH=true
>      - SOLR_SSL_CHECK_PEER_NAME=false
>      - SOLR_URL_SCHEME=https
>
>
>
>
>
> 2019-05-16 22:15:44.999 WARN  (qtp841262455-21) [   ] o.a.s.h.a.AdminHandlersProxy Exception when fetching result from node mynode1_fqdn:8983_solr
> java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://mynode1_fqdn:8983/solr
>                at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:?]
>                at java.util.concurrent.FutureTask.get(FutureTask.java:205) ~[?:?]
>                at org.apache.solr.handler.admin.AdminHandlersProxy.maybeProxyToNodes(AdminHandlersProxy.java:102) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:136) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.handler.admin.InfoHandler.handle(InfoHandler.java:91) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:81) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341) [solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:07]
>                at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1602) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) [jetty-security-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) [jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335) [jetty-rewrite-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.Server.handle(Server.java:502) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) [jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) [jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) [jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
>                at java.lang.Thread.run(Thread.java:834) [?:?]
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://mynode1_fqdn:8983/solr
>                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
>                at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
>                at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
>                ... 1 more
> Caused by: org.apache.http.client.ClientProtocolException
>                at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:187) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
>                at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>                at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
>                at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
>                ... 1 more
> Caused by: org.apache.http.ProtocolException: The server failed to respond with a valid HTTP response
>                at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:149) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) ~[httpcore-4.4.10.jar:4.4.10]
>                at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) ~[httpcore-4.4.10.jar:4.4.10]
>                at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) ~[httpcore-4.4.10.jar:4.4.10]
>                at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) ~[httpcore-4.4.10.jar:4.4.10]
>                at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.6.jar:4.5.6]
>                at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan - 2019-02-23 02:39:09]
>
>
>
> Tony Sargent
> Unix Middle Tier Admin, Web Operations
> 801.442.5579 | [hidden email]<mailto:[hidden email]>