<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 23:06:23 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[FOLIO-516] Error 500&apos;s from Okapi with 5 or more concurrent requests</title>
                <link>https://folio-org.atlassian.net/browse/FOLIO-516</link>
                <project id="10290" key="FOLIO">FOLIO</project>
                    <description>&lt;p&gt;For a while now, I have - on-and-off - seen internal server errors from Okapi, apparently when issuing 5-6 or more concurrent requests to Okapi. &lt;/p&gt;

&lt;p&gt;I&apos;m running folio-backend-auth black box 0.7.2&lt;/p&gt;

&lt;p&gt;I first saw it in UI scan, &lt;em&gt;not&lt;/em&gt; when adding a single item, which would make following requests (each request preceded with an OPTIONS request): &lt;/p&gt;

&lt;p&gt;GET  item-storage&lt;br/&gt;
PUT item-storage&lt;br/&gt;
POST loan-storage&lt;br/&gt;
GET loan-storage&lt;/p&gt;

&lt;p&gt;But when adding two or more items in one go, the issue occurs more often, though not always.&lt;/p&gt;

&lt;p&gt;I&apos;ve then seen it on the users view page two - where we now have about five concurrent GET requests to loans, users, permissions (x2), groups.&lt;/p&gt;

&lt;p&gt;I&apos;ve begun attempts at reproducing with cUR but need to add more concurrent requests to try and imitate the UI actions. &lt;/p&gt;</description>
                <environment></environment>
        <key id="80162">FOLIO-516</key>
            <summary>Error 500&apos;s from Okapi with 5 or more concurrent requests</summary>
                <type id="10001" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium">Bug</type>
                                            <priority id="10000" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p1.svg">P1</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="5f8314dfbdef80006f6f572d">Adam Dickmeiss</assignee>
                                                                <reporter accountid="5bffed5e2434bf3a1a91d37a">Niels Erik Nielsen</reporter>
                                    <labels>
                            <label>sprint11</label>
                    </labels>
                <created>Thu, 23 Mar 2017 11:08:18 +0000</created>
                <updated>Mon, 12 Nov 2018 14:23:27 +0000</updated>
                            <resolved>Tue, 25 Apr 2017 09:37:14 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>10</watches>
                                                    <timespent seconds="138600">4 days, 6 hours, 30 minutes</timespent>
                                <comments>
                                                            <comment id="188885" author="5bffed5e2434bf3a1a91d37a" created="Fri, 24 Mar 2017 14:58:22 +0000"  >&lt;p&gt;&lt;font color=&quot;red&quot;&gt;Update: These errors may have occurred well before the first request resulting in error 500 &amp;#8211; see subsequent comments on this jira.  &lt;/font&gt;Seems to be related to postgreSQL connection errors. Example of one such error:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
2017-03-24 13:02:26,119 INFO  DockerModuleHandle   startContainer
2017-03-24 13:02:26,229 INFO  DockerModuleHandle   startContainer
2017-03-24 13:29:09,673 ERROR PostgreSQLConnection Error with message -&amp;gt; ErrorMessage(fields=Map(Line -&amp;gt; 2875, File -&amp;gt; postgres.c, SQLSTATE -&amp;gt; 57P01, Routine -&amp;gt; ProcessInterrupts, V -&amp;gt; FATAL, Message -&amp;gt; terminating connection due to administrator command, Severity -&amp;gt; FATAL))
2017-03-24 13:29:09,673 ERROR PostgreSQLConnection Error with message -&amp;gt; ErrorMessage(fields=Map(Line -&amp;gt; 2875, File -&amp;gt; postgres.c, SQLSTATE -&amp;gt; 57P01, Routine -&amp;gt; ProcessInterrupts, V -&amp;gt; FATAL, Message -&amp;gt; terminating connection due to administrator command, Severity -&amp;gt; FATAL))
2017-03-24 13:29:09,674 ERROR PostgreSQLConnection Error on connection
com.github.mauricio.async.db.postgresql.exceptions.GenericDatabaseException: ErrorMessage(fields=Map(Line -&amp;gt; 2875, File -&amp;gt; postgres.c, SQLSTATE -&amp;gt; 57P01, Routine -&amp;gt; ProcessInterrupts, V -&amp;gt; FATAL, Message -&amp;gt; terminating connection due to administrator command, Severity -&amp;gt; FATAL))
        at com.github.mauricio.async.db.postgresql.PostgreSQLConnection.onError(PostgreSQLConnection.scala:172)
        at com.github.mauricio.async.db.postgresql.codec.PostgreSQLConnectionHandler.channelRead0(PostgreSQLConnectionHandler.scala:206)
        at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:334)
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:326)
        at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293)
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:334)
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:326)
        at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1320)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:334)
        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:905)
        at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:123)
        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:563)
        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:504)
        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:418)
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:390)
        at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:742)
        at java.lang.&lt;span class=&quot;code-object&quot;&gt;Thread&lt;/span&gt;.run(&lt;span class=&quot;code-object&quot;&gt;Thread&lt;/span&gt;.java:745)
2017-03-24 13:29:09,675 ERROR PostgreSQLConnection Error on connection
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191584" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Sun, 26 Mar 2017 08:14:31 +0000"  >&lt;p&gt;Hi NE&lt;br/&gt;
is there anything else in the logs - maybe before this exception - that you can add to the issue?&lt;/p&gt;</comment>
                                                            <comment id="191585" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 09:05:16 +0000"  >&lt;p&gt;Hmm, hang on, I suspect I may have messed up the time stamps. Those errors &lt;em&gt;could&lt;/em&gt; actual be from before the 500 errors. &lt;/p&gt;

&lt;p&gt;Now I&apos;ve installed a brand new box (0.8.0) - just to make sure the issue isn&apos;t just found in older boxes. I&apos;m uploading the entire log, from the creation of the machine (03-26) and till the 500&apos;s occur (03-27)  &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;/rest/api/3/attachment/content/63962&quot; title=&quot;okapi.log.1 attached to FOLIO-516&quot; data-attachment-type=&quot;file&quot; data-attachment-name=&quot;okapi.log.1&quot; data-media-services-type=&quot;file&quot; data-media-services-id=&quot;fe6c62ca-d2ab-4668-9786-6dae17b71dde&quot; rel=&quot;noreferrer&quot;&gt;okapi.log.1&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt;I subsequently opened the UI page (user view) again, and this time the requests in question succeeded; no additional lines written to the log. &lt;/p&gt;

&lt;p&gt;I will try again later to see if the issue occurs again after a while or not. &lt;/p&gt;</comment>
                                                            <comment id="191586" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 09:41:05 +0000"  >&lt;p&gt;So, for a number of page loads there were no 500 errors, then it occurred again, with just a single line added to the log file it seems: &lt;/p&gt;

&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
2017-03-27 09:32:18,365 ERROR ConnectionBase       java.lang.IllegalArgumentException: invalid version format: 0
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;The new, entire log file:  &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;/rest/api/3/attachment/content/63963&quot; title=&quot;okapi.log.2 attached to FOLIO-516&quot; data-attachment-type=&quot;file&quot; data-attachment-name=&quot;okapi.log.2&quot; data-media-services-type=&quot;file&quot; data-media-services-id=&quot;cdd28f73-50f1-4c0a-80c2-db60e26773e4&quot; rel=&quot;noreferrer&quot;&gt;okapi.log.2&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                                                            <comment id="191587" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 09:47:38 +0000"  >&lt;p&gt;And, it occurred again, but without any additional log lines written to okapi.log. &lt;/p&gt;</comment>
                                                            <comment id="191588" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 09:56:27 +0000"  >&lt;p&gt;And, finally, now the requests succeeded, no additional log lines. &lt;/p&gt;</comment>
                                                            <comment id="191589" author="5bffed52a1b46046f530c8f7" created="Mon, 27 Mar 2017 22:08:01 +0000"  >&lt;p&gt;I am frequently, but intermittently, seeing this on my MacBook, where Stripes is running against v0.8.0 of Wayne&apos;s auth-blackbox. I get it when trying to view a user record, which results in five more or less simultaneous requests. They &lt;em&gt;all&lt;/em&gt; fail with an HTTP 500, and no content in the response &amp;#8211; see the attached screenshot.&lt;/p&gt;

&lt;p&gt;This is a bit of a showstopper for me: I am basically unable to work on the Users module now. Can we at least increase the Magic Number in our Postgres configuration?&lt;/p&gt;</comment>
                                                            <comment id="191590" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 09:18:46 +0000"  >&lt;p&gt;Folks, I have pushed this to P1 as it is now blocking three of us in different ways;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Niels-Erik in doing multiple checkins.&lt;/li&gt;
	&lt;li&gt;Me in working on full-record display of users.&lt;/li&gt;
	&lt;li&gt;Cate in running demos (see 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;STRIPES-270&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/STRIPES-270&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;Permissions: Getting Fetch Errors When Logging In With Create User Permission&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium&quot; /&gt;
            STRIPES-270
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
)&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Can someone please get on it?&lt;/p&gt;</comment>
                                                            <comment id="191591" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 09:45:59 +0000"  >&lt;p&gt;currently downloading latest black box.&lt;br/&gt;
ran jmeter to first test parallel requests directly against mod-users&lt;br/&gt;
10 concurrent users - 25 request made of post / put / get requests - some 400 errors but no 500&lt;br/&gt;
ran again via okapi -&amp;gt; mod-users - alot of 400 errors but no 500&lt;br/&gt;
will run against black box shortly&lt;/p&gt;</comment>
                                                            <comment id="191592" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 09:49:59 +0000"  >&lt;p&gt;Thanks, Shale, great to know you&apos;re on it.&lt;/p&gt;

&lt;p&gt;Yes, this is an annoyingly random issue. I see it often (but not quite &lt;em&gt;all&lt;/em&gt; the time) on my MacBook, but hardly at all on my Ubuntu box. I hope you can track down a reliable way to reproduce!&lt;/p&gt;

&lt;p&gt;(I suppose it&apos;s possible that the problem is in Okapi rather then in mod-users, so it might be worth running your tests through Okapi rather than directly against mod-users. The Postgres errors that NE reported above suggest not, of course; but it could even be that putting the requests through Okapi affects the timing in a way that makes the problem more likely to manifest.)&lt;/p&gt;</comment>
                                                            <comment id="191593" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 09:54:17 +0000"  >&lt;p&gt;yes, i agree, i dont think this is an okapi issue at all, the jmeter test just allows me to bombard the system occasionally to see how it reacts&lt;/p&gt;</comment>
                                                            <comment id="191594" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 11:20:46 +0000"  >&lt;p&gt;Hi Mike&lt;br/&gt;
If you have not restarted the black box - any chance you an take a look at the mod-users logs? it could really shed some light on the situation.&lt;br/&gt;
if it is possible then to get to the logs you would need to&lt;/p&gt;

&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
sudo docker ps
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;


&lt;p&gt;locate the mod-users in the list and get the container id and then&lt;/p&gt;


&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
sudo docker logs hex_container_id
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;


&lt;p&gt;hopefully the module logged the stack traces and then we can see what happened&lt;/p&gt;

&lt;p&gt;i would like to take this opportunity and suggest either forwarding the logs to a file per module in the /var &lt;br/&gt;
or better yet an ELK install that will just aggregate the logs and expose them via http (no parsing of the logs, just expose them) - so it should be very easy to do and will really help to debug future issues &lt;/p&gt;
</comment>
                                                            <comment id="191596" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 11:42:12 +0000"  >&lt;p&gt;I attached &lt;tt&gt;vagrant-docker-logs.txt&lt;/tt&gt;. It doesn&apos;t look very informative to me, but it might be more helpful for you.&lt;/p&gt;</comment>
                                                            <comment id="191597" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 11:44:33 +0000"  >&lt;p&gt;yea, that log is clean (no errors)&lt;/p&gt;</comment>
                                                            <comment id="191598" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 11:47:42 +0000"  >&lt;p&gt;What log was &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5bffed5e2434bf3a1a91d37a&quot; class=&quot;user-hover&quot; rel=&quot;5bffed5e2434bf3a1a91d37a&quot; data-account-id=&quot;5bffed5e2434bf3a1a91d37a&quot; accountid=&quot;5bffed5e2434bf3a1a91d37a&quot; rel=&quot;noreferrer&quot;&gt;Niels Erik Nielsen&lt;/a&gt; looking at earlier in this thread, where he found messages like&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;2017-03-24 13:29:09,673 ERROR PostgreSQLConnection Error with message -&amp;gt; ErrorMessage(fields=Map(Line -&amp;gt; 2875, File -&amp;gt; postgres.c, SQLSTATE -&amp;gt; 57P01, Routine -&amp;gt; ProcessInterrupts, V -&amp;gt; FATAL, Message -&amp;gt; terminating connection due to administrator command, Severity -&amp;gt; FATAL))&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;?&lt;/p&gt;</comment>
                                                            <comment id="191599" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 11:51:23 +0000"  >&lt;p&gt;i believe the okapi.log, no?&lt;/p&gt;</comment>
                                                            <comment id="191600" author="5bffed5e2434bf3a1a91d37a" created="Tue, 28 Mar 2017 11:55:18 +0000"  >&lt;p&gt;Yes, okapi.log.&lt;/p&gt;

&lt;p&gt;But as I mentioned, those errors probably did not occur at the time of the 500s I was seeing, so I simply don&apos;t know if they&apos;re related. &lt;/p&gt;</comment>
                                                            <comment id="191601" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 13:12:29 +0000"  >&lt;p&gt;i have the env set up - stripes sample platform running against the black box - so far, looks solid (tested 100&apos;s of &apos;getting user&apos; manually) - do we have some automation for recording ui so i can let that run in the background?&lt;/p&gt;</comment>
                                                            <comment id="191602" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 13:16:00 +0000"  >&lt;p&gt;I don&apos;t know what there is &amp;#8211; it may be that Wolfram has mode something like this.&lt;/p&gt;

&lt;p&gt;Maybe ask on one of the broader Slack channels?&lt;/p&gt;</comment>
                                                            <comment id="191603" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 13:38:26 +0000"  >&lt;p&gt;replicated, updating soon&lt;/p&gt;</comment>
                                                            <comment id="191604" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 13:53:33 +0000"  >&lt;p&gt;i am not sure this is an RMB issue - i have gone through the logs of all modules and the only module that i see that throws an exception at the time the error occurs is the log belonging to the auth module&lt;/p&gt;

&lt;p&gt; logger.error(&quot;AuthZ&amp;gt; Unable to retrieve permissions for &quot; + username + &quot;: &quot; + res.cause().getMessage());&lt;/p&gt;

&lt;p&gt;AuthZ&amp;gt; Unable to retrieve permissions for diku_admin connection was closed&lt;/p&gt;

&lt;p&gt;the code does not print out the stack trace so this is all that there is - i can continue looking at this but it may be more productive for someone who knows the code better&lt;/p&gt;</comment>
                                                            <comment id="191605" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 13:56:28 +0000"  >&lt;p&gt;That would be Kurt.&lt;/p&gt;</comment>
                                                            <comment id="191606" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 14:02:02 +0000"  >&lt;p&gt;added kurt as watcher&lt;/p&gt;</comment>
                                                            <comment id="191607" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 14:09:42 +0000"  >&lt;p&gt;seems like this is caused when trying to get a users permissions via http from /perms/users/??&lt;br/&gt;
kurt, you probably know better&lt;/p&gt;</comment>
                                                            <comment id="191608" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 28 Mar 2017 14:13:27 +0000"  >&lt;p&gt;Whenever a request comes through, it gets sent to the authtoken module which, in turn, sends a request to the permissions module to determine what the user&apos;s current permissions are, to determine whether or not to route the request through.&lt;/p&gt;

&lt;p&gt;Is the permissions module having an issue with higher volume?&lt;/p&gt;</comment>
                                                            <comment id="191609" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 14:15:57 +0000"  >&lt;p&gt;permission seems ok, i only see 200 satus codes in the log&lt;/p&gt;</comment>
                                                            <comment id="191610" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 14:16:56 +0000"  >&lt;p&gt;response times of the permission mod are under 15 milli per requet&lt;/p&gt;</comment>
                                                            <comment id="191611" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 14:22:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt;, what makes you think the &lt;tt&gt;/perms/users&lt;/tt&gt; request is the guilty one? Whenever I see this, the Users module fires off five requests more or less simultaneously; then they &lt;em&gt;all- block for maybe 10 seconds; then they _all&lt;/em&gt; fail more or less simultaneously with the 500.&lt;/p&gt;</comment>
                                                            <comment id="191612" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 14:26:26 +0000"  >&lt;p&gt;that is what i saw in the code - &lt;br/&gt;
the logs in the mod-users seem ok - i can recheck, they seem ok in the permissions as well. the only place i saw an error was in the mod-auth &lt;br/&gt;
looked in the code at the error and followed it to an http call to /perms/users&lt;/p&gt;</comment>
                                                            <comment id="191613" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 14:28:20 +0000"  >&lt;p&gt;OK, cool.&lt;/p&gt;

&lt;p&gt;Kurt, are you able to reproduce this?&lt;/p&gt;</comment>
                                                            <comment id="191614" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 28 Mar 2017 14:32:28 +0000"  >&lt;p&gt;I haven&apos;t attempted to replicate yet. One thing I&apos;ll mention is that I do tend to get a few initial failures the first time I attempt to use the auth modules, because, for whatever reason, the initial request to the permissions module tends to take longer than the authtoken&apos;s timeout is. Subsequent requests fire fine without issues.&lt;/p&gt;</comment>
                                                            <comment id="191615" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 14:34:31 +0000"  >&lt;p&gt;This is definitely not about initial requests. Until the five-requests-at-once moment, the permissions module has been behaving fine.&lt;/p&gt;</comment>
                                                            <comment id="191616" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 18:05:35 +0000"  >&lt;p&gt;kurt, would it be possible to maybe add to the http connection a timer (before the call and when the callback / exception is called) and print that out so maybe we can see if this is caused by a network hiccups? &lt;/p&gt;</comment>
                                                            <comment id="191617" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 28 Mar 2017 18:07:42 +0000"  >&lt;p&gt;Which connection? Do you mean in the authtoken module?&lt;/p&gt;</comment>
                                                            <comment id="191618" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 28 Mar 2017 18:09:51 +0000"  >&lt;p&gt;yes, i believe this is the 500&lt;/p&gt;


&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
usePermissionsSource.getPermissionsForUser(username).setHandler((AsyncResult&amp;lt;JsonArray&amp;gt; res) -&amp;gt; {

      &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt;(res.failed()) {
        logger.error(&lt;span class=&quot;code-quote&quot;&gt;&quot;AuthZ&amp;gt; Unable to retrieve permissions &lt;span class=&quot;code-keyword&quot;&gt;for&lt;/span&gt; &quot;&lt;/span&gt; + username + &lt;span class=&quot;code-quote&quot;&gt;&quot;: &quot;&lt;/span&gt; + res.cause().getMessage());
        ctx.response()
                .setStatusCode(500)
                &lt;span class=&quot;code-comment&quot;&gt;//.end(&lt;span class=&quot;code-quote&quot;&gt;&quot;Unable to retrieve permissions &lt;span class=&quot;code-keyword&quot;&gt;for&lt;/span&gt; user&quot;&lt;/span&gt;);
&lt;/span&gt;                .end();
        &lt;span class=&quot;code-keyword&quot;&gt;return&lt;/span&gt;;
}
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191619" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 28 Mar 2017 18:12:16 +0000"  >&lt;p&gt;Sure, I&apos;ll add the timer.&lt;/p&gt;</comment>
                                                            <comment id="191620" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 28 Mar 2017 20:23:55 +0000"  >&lt;p&gt;It should report the ms taken in the log now, when the perms lookup fails.&lt;/p&gt;</comment>
                                                            <comment id="191621" author="5c706fbb47a54a6728e59df2" created="Tue, 28 Mar 2017 22:04:39 +0000"  >&lt;p&gt;I&apos;ve released the Vagrant box folio/folio-backend-auth v0.8.1 with this version of the mod-auth modules. To update your box, cd into the directory where you have your Vagrantfile, then:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
$ vagrant box update
$ vagrant destroy
$ vagrant up
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191622" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 06:06:31 +0000"  >&lt;p&gt;NE, Mike,&lt;br/&gt;
I will upgrade and re-test - if you guys can also upgrade it would be great so that if you hit the 500 again we can see if this is indeed a networking issue causing the error&lt;/p&gt;</comment>
                                                            <comment id="191623" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 07:12:26 +0000"  >&lt;p&gt;Will do.&lt;/p&gt;

&lt;p&gt;Reading the comments leading up to this, it seems that what we&apos;re describing here is an update that only gives us improved logging &amp;#8211; am I right? So there is a special reason to think this new auth-blackbox will actually &lt;em&gt;fix&lt;/em&gt; my problem?&lt;/p&gt;</comment>
                                                            <comment id="191624" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 07:18:27 +0000"  >&lt;p&gt;correct, this wont fix the problem, From my standpoint, I would like to log this so that we have verification that this is indeed the issue and that a delay in the http request is causing the 500 error. If this is the case, then how to handle should be discussed. I think Kurt would probably know best what should be done in such a case. Should there be a retry automatically? is there another action that can be taken?&lt;/p&gt;</comment>
                                                            <comment id="191625" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 07:47:50 +0000"  >&lt;p&gt;I downloaded the new VM and am running it now. I can confirm that it doesn&apos;t fix the problem.&lt;/p&gt;

&lt;p&gt;I will attach the new logs. Did I get everything of interest?&lt;/p&gt;</comment>
                                                            <comment id="191626" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 07:54:03 +0000"  >&lt;p&gt;this is the mod-user log, and it looks ok - no errors - i assume you&apos;ve encountered the 500 than, can you attach the mod-auth log? you should see it when running the &lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
sudo docker logs
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
&lt;p&gt; command&lt;/p&gt;</comment>
                                                            <comment id="191627" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 08:00:02 +0000"  >&lt;p&gt;Read on &amp;#8211; the log I posted includes the mod-users, mod-authtoken and mod-permissions logs.&lt;/p&gt;</comment>
                                                            <comment id="191628" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 08:07:06 +0000"  >&lt;p&gt;sorry, missed it - yup this is indeed the problem&lt;/p&gt;

&lt;p&gt;Mar 29, 2017 7:43:18 AM mod-auth-authtoken-module&lt;br/&gt;
SEVERE: AuthZ&amp;gt; Unable to retrieve permissions for diku_admin: Connection was closed request took 10011 ms&lt;br/&gt;
Mar 29, 2017 7:43:18 AM mod-auth-authtoken-module&lt;br/&gt;
SEVERE: AuthZ&amp;gt; Unable to retrieve permissions for diku_admin: Connection was closed request took 10018 ms&lt;br/&gt;
Mar 29, 2017 7:43:18 AM mod-auth-authtoken-module&lt;br/&gt;
SEVERE: AuthZ&amp;gt; Unable to retrieve permissions for diku_admin: Connection was closed request took 10021 ms&lt;br/&gt;
Mar 29, 2017 7:43:18 AM mod-auth-authtoken-module&lt;br/&gt;
SEVERE: AuthZ&amp;gt; Unable to retrieve permissions for diku_admin: Connection was closed request took 10022 ms&lt;br/&gt;
Mar 29, 2017 7:43:18 AM mod-auth-authtoken-module&lt;br/&gt;
SEVERE: AuthZ&amp;gt; Unable to retrieve permissions for diku_admin: Connection was closed request took 10059 ms&lt;/p&gt;


&lt;p&gt;and in the code&lt;/p&gt;


&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
HttpClientOptions options = &lt;span class=&quot;code-keyword&quot;&gt;new&lt;/span&gt; HttpClientOptions();
options.setConnectTimeout(10);
options.setIdleTimeout(10)
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191629" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 08:11:16 +0000"  >&lt;p&gt;Looking at the perm log , it seems that it is responding ok - see the Log entries HTTP_1_1 200 (response code) 730 (response size) after that is the response times which range from 98 - 1379 milliseconds&lt;/p&gt;


&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
[Wed Mar 29 07:42:53 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.PomReader INFO module name: permissions_module, version: 2.2.0-SNAPSHOT 
[Wed Mar 29 07:42:53 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.persist.PostgresClient INFO Using schema: diku 
[Wed Mar 29 07:42:53 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.persist.PostgresClient INFO postgreSQLClientConfig = {&lt;span class=&quot;code-quote&quot;&gt;&quot;host&quot;&lt;/span&gt;:&lt;span class=&quot;code-quote&quot;&gt;&quot;10.0.2.15&quot;&lt;/span&gt;,&lt;span class=&quot;code-quote&quot;&gt;&quot;database&quot;&lt;/span&gt;:&lt;span class=&quot;code-quote&quot;&gt;&quot;mod_auth&quot;&lt;/span&gt;,&lt;span class=&quot;code-quote&quot;&gt;&quot;password&quot;&lt;/span&gt;:&lt;span class=&quot;code-quote&quot;&gt;&quot;...&quot;&lt;/span&gt;,&lt;span class=&quot;code-quote&quot;&gt;&quot;port&quot;&lt;/span&gt;:5432,&lt;span class=&quot;code-quote&quot;&gt;&quot;username&quot;&lt;/span&gt;:&lt;span class=&quot;code-quote&quot;&gt;&quot;diku_permissions_module&quot;&lt;/span&gt;} 
[Wed Mar 29 07:42:53 UTC 2017] io.vertx.ext.asyncsql.impl.PostgreSQLClientImpl INFO Creating configuration &lt;span class=&quot;code-keyword&quot;&gt;for&lt;/span&gt; 10.0.2.15:5432 
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51264 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 1379 tid=diku OK  
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51265 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 826 tid=diku OK  
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:42:54 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51264 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt;&amp;amp;full=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 8768 111 tid=diku OK  
[Wed Mar 29 07:42:59 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:42:59 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51265 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 81 tid=diku OK  
[Wed Mar 29 07:42:59 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:42:59 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51264 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 69 tid=diku OK  
[Wed Mar 29 07:43:00 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:00 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51265 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 41 tid=diku OK  
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51264 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 98 tid=diku OK  
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51265 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 126 tid=diku OK  
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51290 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 139 tid=diku OK  
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51288 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 199 tid=diku OK  
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle start  invoking getPermsUsersByUsernamePermissions 
[Wed Mar 29 07:43:18 UTC 2017] org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.tools.utils.LogUtil INFO 10.0.2.15:51289 GET /perms/users/diku_admin/permissions expanded=&lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt; HTTP_1_1 200 730 206 tid=diku OK 
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
</comment>
                                                            <comment id="191630" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 08:17:13 +0000"  >&lt;p&gt;What we need here is a request ID that is generated by Okapi and is maintained in the header across all requests so we can identify the exact call and the exact entry for that call across modules. i have mentioned this in the past, and i think this may happen - i wouldnt mind a quick and dirty implementation for now, could help us almost immediately  to debug problems&lt;/p&gt;</comment>
                                                            <comment id="191631" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 08:39:28 +0000"  >&lt;p&gt;Hi Kurt,&lt;br/&gt;
if an exception is thrown in the body handler below while parsing the json , so that the future.complete() isnt reached - can this cause this? (just speculating)&lt;/p&gt;


&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
    HttpClientRequest req = client.getAbs(requestUrl, res-&amp;gt; {
      &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt;(res.statusCode() == 200) {
        res.bodyHandler(res2 -&amp;gt; {
          JsonObject permissionsObject = &lt;span class=&quot;code-keyword&quot;&gt;new&lt;/span&gt; JsonObject(res2.toString());
          logger.debug(&lt;span class=&quot;code-quote&quot;&gt;&quot;Got permissions: &quot;&lt;/span&gt; + permissionsObject.getJsonArray(&lt;span class=&quot;code-quote&quot;&gt;&quot;permissionNames&quot;&lt;/span&gt;).encodePrettily());
          &lt;span class=&quot;code-keyword&quot;&gt;future&lt;/span&gt;.complete(permissionsObject.getJsonArray(&lt;span class=&quot;code-quote&quot;&gt;&quot;permissionNames&quot;&lt;/span&gt;));
        });
}
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191632" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 08:45:19 +0000"  >&lt;p&gt;Last comment - sorry for the spam&lt;/p&gt;

&lt;p&gt;RMB modules have a URL endpoint to turn on debugging for any class / package - the RMB (if in debug mode on the RestVerticle.class) will print out the incoming request&apos;s body and i believe the outgoing response body - so we can also see if the in/outs are as we expect - this is in the /admin endpoints - and i have no clue how to run those through okapi - so if someone knows , we can turn this on in the perm module&lt;/p&gt;</comment>
                                                            <comment id="191633" author="5f8314dfbdef80006f6f572d" created="Wed, 29 Mar 2017 08:55:07 +0000"  >&lt;p&gt;Uncaught exceptions are mostly seen as hangs for Vert.x .. Has bit me several times!&lt;/p&gt;</comment>
                                                            <comment id="191634" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Wed, 29 Mar 2017 09:01:24 +0000"  >&lt;p&gt;The /admin endpoints can not be invoked through Okapi directly. Either you have to bypass Okapi and call the module directly, or you have to fake a ModuleDescriptor that defines that endpoint - but only for one module in the whole installation.  We have discussed this before, and it can be solved, but there are some difficulties with multiple instances and multiple modules.&lt;/p&gt;

&lt;p&gt;As to the request ID, it would not be too hard to implement. It would require a small change in the auth module, and one in Okapi. The modules would not have to do more than they do now - pass the X-Okapi-Token to any request they make. They would receive an extra header X-Okapi-Request-Id which they could log as they please. Okapi could include that in its logs too. (I thought I had filed an issue for this, but can not find it)&lt;/p&gt;</comment>
                                                            <comment id="191635" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 09:11:24 +0000"  >&lt;p&gt;I think the Id would really prove valuable since we&apos;re already crossing module boundaries &lt;/p&gt;</comment>
                                                            <comment id="191636" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 10:42:00 +0000"  >&lt;blockquote&gt;&lt;p&gt;Uncaught exceptions are mostly seen as hangs for Vert.x&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;That is &lt;em&gt;awful&lt;/em&gt;!&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;The /admin endpoints can not be invoked through Okapi directly. Either you have to bypass Okapi and call the module directly, or you have to fake a ModuleDescriptor that defines that endpoint - but only for one module in the whole installation. We have discussed this before, and it can be solved, but there are some difficulties with multiple instances and multiple modules.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I think it would be &lt;em&gt;very&lt;/em&gt; well worth coming up with a general solution to this in future, although I realise it&apos;s not going to get done right now. For example, some special-casing in Okapi where &lt;tt&gt;/_/admin/INSTANCEID&lt;/tt&gt; is proxied to &lt;tt&gt;/admin&lt;/tt&gt; on the module with the specified INSTANCEID.&lt;/p&gt;

&lt;p&gt;But, folks &amp;#8211; what are the prospects for fixing the present issue before we have mechanisms like this and the request-ID?&lt;/p&gt;</comment>
                                                            <comment id="191637" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 29 Mar 2017 12:08:02 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; Thanks for the catch about the possible exception handling. Tightening that block up (though I don&apos;t know if it is the culprit here or not.)&lt;/p&gt;</comment>
                                                            <comment id="191638" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 12:11:59 +0000"  >&lt;p&gt;np, not sure either - i have set up my env with debug log level on a couple of packages in the mod-perms and see if i can see what is returned when the error occurs&lt;/p&gt;</comment>
                                                            <comment id="191639" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 12:20:19 +0000"  >&lt;p&gt;Mike, NE, Kurt,&lt;br/&gt;
i have been pressing for a while with no reproduce, can you guys maybe send the following urls to the perms module (you need to replace the port with the one that appears in the output of docker ps) - dont know if there is enough debug info in there - but it is much more then now - so could help&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.rest.RestVerticle&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.rest.RestVerticle&lt;/a&gt;&lt;br/&gt;
&lt;a href=&quot;http://localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.rest.impl&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.rest.impl&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="191640" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 29 Mar 2017 12:47:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; This is tangential to the discussion, but how hard would it be to give the RMB an option to enable debug logging on the command line string, as opposed to requiring a request to be sent to the /admin endpoint?&lt;/p&gt;</comment>
                                                            <comment id="191641" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 12:51:08 +0000"  >&lt;p&gt;i can add it - but i might be missing your angle on this, you mean to have it startup in debug mode for testing purposes?&lt;br/&gt;
if so, maybe wayne can add a call (dont know if it is too complicated) - to the endpoint when launching?&lt;br/&gt;
but it is easy enough to add the command line as well&lt;/p&gt;</comment>
                                                            <comment id="191642" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 29 Mar 2017 12:56:10 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; Yes, that&apos;s exactly what I want...to be able to start in debug mode with just a flag in the deployment descriptor. Yes, I can hit the endpoint manually by looking it up in Okapi, but it&apos;s kind of cumbersome. If that&apos;s something we can do, I&apos;ll open a separate issue for it so we don&apos;t derail this one.&lt;/p&gt;</comment>
                                                            <comment id="191643" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 12:59:43 +0000"  >&lt;p&gt;Clarification: those HTTP requests that &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; mentioned are PUTs with no payload, and they only work on RMB-based modules &amp;#8211; so in this case, mod-users and mod-permissions, but not mod-authcheck.&lt;/p&gt;</comment>
                                                            <comment id="191644" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 13:05:38 +0000"  >&lt;p&gt;Here is the result of turning up the log levels. (Next I will post the resulting logs.)&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
vagrant@jessie:~$ curl -w &lt;span class=&quot;code-quote&quot;&gt;&apos;\n&apos;&lt;/span&gt; -X PUT -d &lt;span class=&quot;code-quote&quot;&gt;&apos;&apos; &apos;&lt;/span&gt;http:&lt;span class=&quot;code-comment&quot;&gt;//localhost:9133/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle&apos;
&lt;/span&gt;{
  &lt;span class=&quot;code-quote&quot;&gt;&quot;map&quot;&lt;/span&gt; : {
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;
  },
  &lt;span class=&quot;code-quote&quot;&gt;&quot;empty&quot;&lt;/span&gt; : &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;
}
vagrant@jessie:~$ curl -w &lt;span class=&quot;code-quote&quot;&gt;&apos;\n&apos;&lt;/span&gt; -X PUT -d &lt;span class=&quot;code-quote&quot;&gt;&apos;&apos; &apos;&lt;/span&gt;http:&lt;span class=&quot;code-comment&quot;&gt;//localhost:9133/admin/loglevel?level=FINE&amp;amp;java_package=java_package=org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl&apos;
&lt;/span&gt;{
  &lt;span class=&quot;code-quote&quot;&gt;&quot;map&quot;&lt;/span&gt; : { },
  &lt;span class=&quot;code-quote&quot;&gt;&quot;empty&quot;&lt;/span&gt; : &lt;span class=&quot;code-keyword&quot;&gt;true&lt;/span&gt;
}
vagrant@jessie:~$ 
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
&lt;p&gt;That second one doesn&apos;t look right to me.&lt;/p&gt;</comment>
                                                            <comment id="191645" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 13:06:16 +0000"  >&lt;p&gt;Oh, I see what I did wrong. Here you go:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
vagrant@jessie:~$ curl -w &lt;span class=&quot;code-quote&quot;&gt;&apos;\n&apos;&lt;/span&gt; -X PUT -d &lt;span class=&quot;code-quote&quot;&gt;&apos;&apos; &apos;&lt;/span&gt;http:&lt;span class=&quot;code-comment&quot;&gt;//localhost:9133/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl&apos;
&lt;/span&gt;{
  &lt;span class=&quot;code-quote&quot;&gt;&quot;map&quot;&lt;/span&gt; : {
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.AdminAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.UsersAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.ProcessUploads&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.TenantAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.JobAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.JobsRunner&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.UserGroupAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;
  },
  &lt;span class=&quot;code-quote&quot;&gt;&quot;empty&quot;&lt;/span&gt; : &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;
}
vagrant@jessie:~$ 
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191646" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 13:08:29 +0000"  >&lt;p&gt;That was turning the logging up for mod-users; here&apos;s the result of turning it up for mod-permissions:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
vagrant@jessie:~$ curl -w &lt;span class=&quot;code-quote&quot;&gt;&apos;\n&apos;&lt;/span&gt; -X PUT -d &lt;span class=&quot;code-quote&quot;&gt;&apos;&apos; &apos;&lt;/span&gt;http:&lt;span class=&quot;code-comment&quot;&gt;//localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle&apos;
&lt;/span&gt;{
  &lt;span class=&quot;code-quote&quot;&gt;&quot;map&quot;&lt;/span&gt; : {
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.RestVerticle&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;
  },
  &lt;span class=&quot;code-quote&quot;&gt;&quot;empty&quot;&lt;/span&gt; : &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;
}
vagrant@jessie:~$ curl -w &lt;span class=&quot;code-quote&quot;&gt;&apos;\n&apos;&lt;/span&gt; -X PUT -d &lt;span class=&quot;code-quote&quot;&gt;&apos;&apos; &apos;&lt;/span&gt;http:&lt;span class=&quot;code-comment&quot;&gt;//localhost:9139/admin/loglevel?level=FINE&amp;amp;java_package=org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl&apos;
&lt;/span&gt;{
  &lt;span class=&quot;code-quote&quot;&gt;&quot;map&quot;&lt;/span&gt; : {
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.PermsAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.AdminAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.ProcessUploads&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.TenantAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.JobAPI&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;,
    &lt;span class=&quot;code-quote&quot;&gt;&quot;org.folio.&lt;span class=&quot;code-keyword&quot;&gt;rest&lt;/span&gt;.impl.JobsRunner&quot;&lt;/span&gt; : &lt;span class=&quot;code-quote&quot;&gt;&quot;FINE&quot;&lt;/span&gt;
  },
  &lt;span class=&quot;code-quote&quot;&gt;&quot;empty&quot;&lt;/span&gt; : &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;
}
vagrant@jessie:~$ 
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="191647" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 13:19:04 +0000"  >&lt;p&gt;I attached a transcript of the mod-users and mod-permissions FINE logging. Hope it helps!&lt;/p&gt;

&lt;p&gt;BTW., a big thank you, guys &amp;#8211; I really appreciate you all putting in the hard yards on this.&lt;/p&gt;</comment>
                                                            <comment id="191648" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 13:56:19 +0000"  >&lt;p&gt;Kurt, i have to head out, that once in a year day has come and i need to prepare. I want to push a fix into master that will print out the returned json in debug mode (seems this is broken for some reason) - will do if you think that can help understand the issue&lt;/p&gt;</comment>
                                                            <comment id="191649" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 29 Mar 2017 13:59:11 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; Returned JSON from which module? I don&apos;t want to keep you from your observances.&lt;/p&gt;</comment>
                                                            <comment id="191650" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 29 Mar 2017 14:08:35 +0000"  >&lt;p&gt;just my birthday, no biggie, adding to the RMB - but i can upgrade perms to use the RMB version and then you can see the outputs , relevant?&lt;/p&gt;</comment>
                                                            <comment id="191651" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 29 Mar 2017 14:24:19 +0000"  >&lt;p&gt;I have some other diagnostics I want to try, so I&apos;d say you can wait until tomorrow to worry about the push.&lt;/p&gt;</comment>
                                                            <comment id="191652" author="5bffed52a1b46046f530c8f7" created="Tue, 4 Apr 2017 14:01:04 +0000"  >&lt;p&gt;Folks, we really really need to get this fixed. As things stand FOLIO services are Just Plain Broken.&lt;/p&gt;

&lt;p&gt;The most recent random manifestation of the problem is preventing Jeremy from finishing up the work on 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;LIBAPP-83&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/LIBAPP-83&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;CRUD Permission Sets v1&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium&quot; /&gt;
            LIBAPP-83
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 for the demo we&apos;re supposed to be doing in literally one hour from now.&lt;/p&gt;</comment>
                                                            <comment id="191653" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 5 Apr 2017 10:25:30 +0000"  >&lt;p&gt;Hi Kurt,&lt;br/&gt;
 are you looking into this or should i?&lt;/p&gt;</comment>
                                                            <comment id="191654" author="5bffed52a1b46046f530c8f7" created="Fri, 7 Apr 2017 14:27:48 +0000"  >&lt;p&gt;Kurt, are you looking into this? &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; class=&quot;user-hover&quot; rel=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; data-account-id=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; accountid=&quot;712020:32bb56ac-50e7-4787-b4af-ed3089d9401c&quot; rel=&quot;noreferrer&quot;&gt;shale99&lt;/a&gt; is not, he&apos;s out today.&lt;/p&gt;</comment>
                                                            <comment id="191655" author="5c38e8d616ac1e4f7cbc660a" created="Fri, 7 Apr 2017 14:31:15 +0000"  >&lt;p&gt;I have started some basic investigation, but I have nothing conclusive yet. I will try to carve out some time this afternoon to investigate further.&lt;/p&gt;</comment>
                                                            <comment id="191656" author="5bffed52a1b46046f530c8f7" created="Fri, 14 Apr 2017 16:22:27 +0000"  >&lt;p&gt;CAN SOMEONE PLEASE FIX THIS?&lt;/p&gt;

&lt;p&gt;It&apos;s been at P1 for a couple of weeks. It is simply not possible to coherently do front-end development work when the back-end keeps randomly getting into a state where every request fails for no reason.&lt;/p&gt;</comment>
                                                            <comment id="191657" author="5c38e8d616ac1e4f7cbc660a" created="Fri, 14 Apr 2017 16:30:15 +0000"  >&lt;p&gt;I did some investigation and I think it might be due to timeouts in the authtoken module while looking up permissions. Trying with a longer timeout threshold and also looking into ways to improve the lookup performance.&lt;/p&gt;</comment>
                                                            <comment id="191658" author="5bffed52a1b46046f530c8f7" created="Fri, 14 Apr 2017 16:33:35 +0000"  >&lt;p&gt;Thanks, but I don&apos;t &lt;em&gt;think&lt;/em&gt; it&apos;s timeouts. All this is happening very fast.&lt;/p&gt;

&lt;p&gt;Surely stage one is just to ensure you never ever return a 5xx without some diagnostic text in the response body?&lt;/p&gt;</comment>
                                                            <comment id="191659" author="5c38e8d616ac1e4f7cbc660a" created="Fri, 14 Apr 2017 16:44:21 +0000"  >&lt;p&gt;I&apos;ve been avoiding passing specific information about the backend into the response body for 500&apos;s and instead writing the information out to logs when server errors occur.&lt;/p&gt;

&lt;p&gt;Maybe a debug flag in which the information gets sent to the response would be useful in cases like this.&lt;/p&gt;</comment>
                                                            <comment id="191660" author="5bffed52a1b46046f530c8f7" created="Fri, 14 Apr 2017 20:04:41 +0000"  >&lt;p&gt;Why either/or? Why not just emit the messages in the HTTP responses &lt;em&gt;and&lt;/em&gt; log them?&lt;/p&gt;</comment>
                                                            <comment id="191661" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Tue, 18 Apr 2017 12:33:46 +0000"  >&lt;p&gt;hi kurt,&lt;br/&gt;
i believe the culprit is the following function, getPermsUsersByUsernamePermissions() in the permissions-module. i would like to add some logging info to that function if thats ok,&lt;br/&gt;
there is also a recursive function called from within that function getExpandedPermissions(), not familiar with the biz logic - but maybe something there?&lt;/p&gt;</comment>
                                                            <comment id="191662" author="5c38e8d616ac1e4f7cbc660a" created="Tue, 18 Apr 2017 13:09:31 +0000"  >&lt;p&gt;Fine by me. Let me know what you turn up.&lt;/p&gt;</comment>
                                                            <comment id="191663" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Wed, 19 Apr 2017 06:31:45 +0000"  >&lt;p&gt;Hey guys,&lt;br/&gt;
so apologies, i thought this was being handled so i did not continue to look into this. I understood from adam that was not the case and that he is checking a few things. If that does not turn up anything i will dive back into this.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt; - would it be possible to start java processes within docker with debugging enabled - i think this makes sense for future cases as well, basically the following params should be added to the java command line and this will allow connecting with a remote debugger to debug the java code running in a docker container (need to map the port)&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
&lt;p&gt;(the port would be different for each of the started java modules in docker)&lt;/p&gt;</comment>
                                                            <comment id="191664" author="5f8314dfbdef80006f6f572d" created="Wed, 19 Apr 2017 13:52:28 +0000"  >&lt;p&gt;The 500-problem has now been reproduced with using curl exclusively. I think we&apos;re closer to nailing this one down. This is not the startup problem.. It really seems to happen when we fire concurrent requests to the system.&lt;/p&gt;</comment>
                                                            <comment id="191666" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 19 Apr 2017 13:54:31 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f8314dfbdef80006f6f572d&quot; class=&quot;user-hover&quot; rel=&quot;5f8314dfbdef80006f6f572d&quot; data-account-id=&quot;5f8314dfbdef80006f6f572d&quot; accountid=&quot;5f8314dfbdef80006f6f572d&quot; rel=&quot;noreferrer&quot;&gt;Adam Dickmeiss&lt;/a&gt; Can you share your curl recipe for reproducing the issue? I&apos;d like to try to witness it locally.&lt;/p&gt;</comment>
                                                            <comment id="191668" author="5bffed52a1b46046f530c8f7" created="Wed, 19 Apr 2017 14:08:55 +0000"  >&lt;p&gt;That is &lt;em&gt;great&lt;/em&gt; news, Adam!&lt;/p&gt;</comment>
                                                            <comment id="191669" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Wed, 19 Apr 2017 14:39:49 +0000"  >&lt;p&gt;I think we reproduced the issue in a simplified form, using Okapi in dev mode and the Perl sample module. That means no databases were involved at all, no docker images, nothing but Okapi and the perl script. The problem only happens when the module makes a request to some module - in this case itself.&lt;/p&gt;

&lt;p&gt;More info to follow, probably tomorrow&lt;/p&gt;
</comment>
                                                            <comment id="191670" author="5c38e8d616ac1e4f7cbc660a" created="Wed, 19 Apr 2017 14:41:31 +0000"  >&lt;p&gt;Augh! Don&apos;t leave me in suspense! At least give me a bash script or something to reproduce the issue.&lt;/p&gt;</comment>
                                                            <comment id="191671" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Wed, 19 Apr 2017 14:48:07 +0000"  >&lt;p&gt;Console 1:&lt;br/&gt;
java -Dloglevel=DEBUG -jar okapi-core/target/okapi-core-fat.jar dev&lt;/p&gt;

&lt;p&gt;Console 2:&lt;br/&gt;
cd ~/proj/folio/folio-sample-modules/simple-perl&lt;br/&gt;
./example.sh&lt;/p&gt;

&lt;p&gt;Script to run on test consoles:&lt;br/&gt;
while curl -f -H &quot;X-Okapi-Tenant: testlib&quot;  &lt;a href=&quot;http://localhost:9130/simple&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://localhost:9130/simple&lt;/a&gt;&lt;br/&gt;
do&lt;br/&gt;
 echo -n `date`&lt;br/&gt;
 sleep 0.01&lt;br/&gt;
done&lt;/p&gt;

&lt;p&gt;Start about 10 of those consoles, and pay attention to the time stamp. At some point things stop moving. Ctrl-C on Okapi, and the scripts die nicely.&lt;/p&gt;</comment>
                                                            <comment id="191672" author="5f8314dfbdef80006f6f572d" created="Thu, 20 Apr 2017 08:47:06 +0000"  >&lt;p&gt;Looks like something we can reproduce with Okapi so assigning to me and Heikki.&lt;/p&gt;</comment>
                                                            <comment id="191673" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Thu, 20 Apr 2017 09:13:59 +0000"  >&lt;p&gt;With a bit bigger script, I can start 20 clients. As long as the requests make another request, the system freezes almost immediately. If I make a request to /hello instead, the perl module makes no further requests, and then thing seems to run happily with 50 requests.&lt;/p&gt;</comment>
                                                            <comment id="191674" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Thu, 20 Apr 2017 09:36:54 +0000"  >&lt;p&gt;We have it. When Okapi makes a request to a module, it uses a global HttpClient to get a connection object. The HttpClient has a pool of connections, and if all are in use, it waits until one is free. But since the module will not finish its response until it has made its own call to Okapi, the pool has one less connection free. Repeat with many parallel requests, and there are no free connections in the pool to serve the request from the module, and the whole system hangs. A classical deadlock.&lt;/p&gt;

&lt;p&gt;The solution is to use a HttpClient for each request, so the request from the module gets its own HttpClient, and its own connection pool, and will not need to wait for any other request to finish.&lt;/p&gt;

&lt;p&gt;Expect a new Okapi release later today.&lt;/p&gt;</comment>
                                                            <comment id="191675" author="5f8314dfbdef80006f6f572d" created="Thu, 20 Apr 2017 10:38:34 +0000"  >&lt;p&gt;Okapi 1.2.4 is out and a bug 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;OKAPI-312&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/OKAPI-312&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;hang in recursive calls for parallel requests&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium&quot; /&gt;
            OKAPI-312
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 was made the issue or a subset thereof (there might be other issues!). It is of course interesting whehter 1.2.4 will make the problem go away as far as UI development is concerned.&lt;/p&gt;</comment>
                                                            <comment id="191676" author="5bffed52a1b46046f530c8f7" created="Thu, 20 Apr 2017 10:45:54 +0000"  >&lt;p&gt;This is the absolutely best news! Thanks for chasing it through.&lt;/p&gt;</comment>
                                                            <comment id="191677" author="5bffed52a1b46046f530c8f7" created="Thu, 20 Apr 2017 10:46:44 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt; Can we please have a new folio-backend-auth that includes this? Thanks!&lt;/p&gt;</comment>
                                                            <comment id="191678" author="5bffed5e2434bf3a1a91d37a" created="Thu, 20 Apr 2017 15:19:21 +0000"  >&lt;p&gt;Most bugs are just annoying. Intriguing bugs are rare &lt;img class=&quot;emoticon&quot; src=&quot;/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt; &lt;/p&gt;</comment>
                                                            <comment id="191679" author="5f8314dfbdef80006f6f572d" created="Tue, 25 Apr 2017 09:15:58 +0000"  >&lt;p&gt;Can UI people tell us if this problem seems fixed.. If yes, we&apos;ll close this one.&lt;/p&gt;</comment>
                                                            <comment id="191680" author="5bffed52a1b46046f530c8f7" created="Tue, 25 Apr 2017 09:20:04 +0000"  >&lt;p&gt;I have not seen a mysterious 500 since the release of folio-backend-auth v0.10.0. So I think we can close this, at least provisionally &amp;#8211; we can always re-open if it crops up again.&lt;/p&gt;</comment>
                                                            <comment id="191681" author="712020:38d1a08f-86a8-4df2-9191-239b16b0a81a" created="Wed, 26 Apr 2017 08:13:10 +0000"  >&lt;p&gt;I think I was still able to provoke a few 500&apos;s, but on &lt;b&gt;much&lt;/b&gt; heavier loading. Something like 50 clients hammering the system as fast as they can, running for tens of minutes before I saw the first. Probably not worth worrying about at this point, but I wanted to record the fact.  The loop.sh script with the folio-sample-modules/simple-perl is what I used for the test.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10000">
                    <name>Blocks</name>
                                            <outwardlinks description="blocks">
                                        <issuelink>
            <issuekey id="61116">STRIPES-324</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10002">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="61179">STRIPES-270</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                                                <inwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="53581">OKAPI-312</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="53552">OKAPI-370</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="61117">STRIPES-325</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="63961" name="fine-logs-transcript.txt" size="62656" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 13:18:27 +0000"/>
                            <attachment id="63962" name="okapi.log.1" size="22340" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 08:57:46 +0000"/>
                            <attachment id="63963" name="okapi.log.2" size="22454" author="5bffed5e2434bf3a1a91d37a" created="Mon, 27 Mar 2017 09:40:50 +0000"/>
                            <attachment id="63964" name="vagrant-docker-logs.txt" size="31374" author="5bffed52a1b46046f530c8f7" created="Tue, 28 Mar 2017 11:41:17 +0000"/>
                            <attachment id="63965" name="vagrant-logs-new-box.txt" size="15475" author="5bffed52a1b46046f530c8f7" created="Wed, 29 Mar 2017 07:48:11 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|hzxlsv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Sun, 26 Mar 2017 08:14:31 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>