<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 23:09:51 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[FOLIO-968] Container RAM usage is getting out of control on blackbox VMs</title>
                <link>https://folio-org.atlassian.net/browse/FOLIO-968</link>
                <project id="10290" key="FOLIO">FOLIO</project>
                    <description>&lt;p&gt;Each container wants something like 300MB of RAM. This is a little unreasonable, with the number of containers we&apos;re trying to run now. Vagrant boxes produced by CI (and folio-testing-backend01, maybe) are running into OOM issues.&lt;/p&gt;</description>
                <environment></environment>
        <key id="80428">FOLIO-968</key>
            <summary>Container RAM usage is getting out of control on blackbox VMs</summary>
                <type id="10001" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium">Bug</type>
                                            <priority id="10001" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p2.svg">P2</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="5c706fbb47a54a6728e59df2">Wayne Schneider</assignee>
                                                                <reporter accountid="5c706fbb47a54a6728e59df2">Wayne Schneider</reporter>
                                    <labels>
                            <label>ci</label>
                            <label>sprint28</label>
                            <label>sprint29</label>
                            <label>sprint35</label>
                            <label>sprint36</label>
                            <label>sprint37</label>
                            <label>sprint48</label>
                    </labels>
                <created>Wed, 6 Dec 2017 17:57:47 +0000</created>
                <updated>Mon, 12 Nov 2018 14:23:52 +0000</updated>
                            <resolved>Wed, 3 Oct 2018 15:23:15 +0000</resolved>
                                                                    <component>Continuous Integration</component>
                        <due></due>
                            <votes>0</votes>
                                    <watches>7</watches>
                                                    <timespent seconds="27000">7 hours, 30 minutes</timespent>
                                <comments>
                                                            <comment id="190945" author="5c706fbb47a54a6728e59df2" created="Wed, 6 Dec 2017 17:59:55 +0000"  >&lt;p&gt;Top from a recent &quot;snapshot&quot; VM sorted by memory usage shows:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
1158 folio     20   0 3516332 270676  20460 S   0.0  6.7   0:06.83 java                                                       
 1923 folio     20   0 3508048 270624  22272 S   0.3  6.7   0:05.62 java                                                       
 1506 folio     20   0 3506624 233480  20836 S   0.0  5.8   0:05.39 java                                                       
 2195 folio     20   0 3504180 226044  20512 S   0.3  5.6   0:04.20 java                                                       
 2102 folio     20   0 3504204 221220  20324 S   0.0  5.5   0:04.90 java                                                       
 1241 folio     20   0 3506496 218076  20560 S   0.0  5.4   0:05.20 java                                                       
  819 okapi     20   0 3509932 201276  18616 S   0.3  5.0   0:04.63 java                                                       
 2367 folio     20   0 3504472 198380  20788 S   0.0  4.9   0:04.31 java                                                       
 2014 folio     20   0 3504224 197752  20400 S   0.0  4.9   0:03.90 java                                                       
 1326 folio     20   0 3504452 193680  20748 S   0.0  4.8   0:04.00 java                                                       
 2277 folio     20   0 3504452 192240  20720 S   0.3  4.7   0:03.63 java                                                       
 1408 folio     20   0 3495496  75132  17872 S   0.3  1.9   0:01.58 java                                                       
 1683 folio     20   0 3493900  74584  17556 S   0.3  1.8   0:01.46 java                                                       
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="190949" author="5c706fbb47a54a6728e59df2" created="Wed, 6 Dec 2017 18:02:24 +0000"  >&lt;p&gt;Possible useful resources:&lt;br/&gt;
&lt;a href=&quot;https://developers.redhat.com/blog/2017/03/14/java-inside-docker/&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://developers.redhat.com/blog/2017/03/14/java-inside-docker/&lt;/a&gt;&lt;br/&gt;
the &#8220;make JVM respect CPU and RAM limits&#8221; section of &lt;a href=&quot;https://hub.docker.com/r/library/openjdk/&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://hub.docker.com/r/library/openjdk/&lt;/a&gt; suggests there are some specialist parameters for running inside a container&lt;/p&gt;</comment>
                                                            <comment id="190952" author="5f9abc1eb45b2e007453f423" created="Wed, 6 Dec 2017 20:13:10 +0000"  >&lt;p&gt;Some of the issues can be partially addressed by completing 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;FOLIO-886&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/FOLIO-886&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;add java remote debugging as standard for docker java images&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium&quot; /&gt;
            FOLIO-886
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 and utilizing a new base Docker image for server-0side modules that has facilities for managing JVM memory in containers properly.  &lt;/p&gt;</comment>
                                                            <comment id="190954" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Thu, 7 Dec 2017 06:13:21 +0000"  >&lt;p&gt; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;64003_thumb&quot; href=&quot;/rest/api/3/attachment/content/64003&quot; title=&quot;screenshot-1.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;64003&quot; file-preview-title=&quot;screenshot-1.png&quot;&gt;&lt;jira-attachment-thumbnail url=&quot;https://folio-org.atlassian.net/rest/api/3/attachment/thumbnail/64003?default=false&quot; jira-url=&quot;https://folio-org.atlassian.net/rest/api/3/attachment/thumbnail/64003&quot; filename=&quot;screenshot-1.png&quot;&gt;&lt;img src=&quot;https://folio-org.atlassian.net/rest/api/3/attachment/thumbnail/64003&quot; data-attachment-name=&quot;screenshot-1.png&quot; data-attachment-type=&quot;thumbnail&quot; data-media-services-id=&quot;50e148be-22f1-4ff8-8018-aea9a435640c&quot; data-media-services-type=&quot;file&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/jira-attachment-thumbnail&gt;&lt;/a&gt;&lt;/span&gt; &lt;br/&gt;
i&apos;ve uploaded the output from `/admin/memory` for one of the modules (what &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt; sent on slack) - if you look at the summary (the last line) it indicates that the OS has committed 166MB to the JVM and that the JVM is using about 106MB - this is heap memory only (which in java apps is usually the bulk of the allocated memory) whereas the non-heap is at around 50MB - we can potentially do some tuning to cut down a couple of 10&apos;s of MBs, but i am not sure it is worth the effort at this point as this amount of memory isnt considered very high. so while we may be able to reduce the ~280MB , i wouldnt expect this to drop too much&lt;/p&gt;</comment>
                                                            <comment id="190957" author="557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d" created="Thu, 7 Dec 2017 13:15:33 +0000"  >&lt;p&gt;Assigning to &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt; while &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt; is away. John can you check if you can deal with this by allocating more resources to the box. Also if there&apos;s anything specific that should be done to decrease resource usage for specific module. Which modules are the heavy resource consumers? &lt;/p&gt;</comment>
                                                            <comment id="190959" author="5c706fbb47a54a6728e59df2" created="Mon, 18 Dec 2017 23:01:21 +0000"  >&lt;p&gt;One thing to try &amp;#8211; set &lt;tt&gt;-Xmx&lt;/tt&gt; for all Java-based modules. A look at &lt;tt&gt;/admin/memory&lt;/tt&gt; on all the RMB-based modules suggests that 256m is a reasonable baseline; mod-permissions and mod-inventory-storage may need to be set to 512m, and mod-users to 384m.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt; also asked that a couple of debugging flags (&lt;tt&gt;-XX:+PrintFlagsFinal&lt;/tt&gt; and &lt;tt&gt;-XX:+PrintGCDetails&lt;/tt&gt;) be set. The only problem is that there appears to be a bug in the docker_entrypoint.sh script, so that you can only set one flag in the JAVA_OPTS environment variable when launching the docker container. That may be need to be filed as a separate issue.&lt;/p&gt;</comment>
                                                            <comment id="190961" author="5c706fbb47a54a6728e59df2" created="Tue, 19 Dec 2017 04:52:33 +0000"  >&lt;p&gt;Limiting heap size does seem to help (at least marginally), without obviously affecting performance. With the heap size limit, it appears we can configure the Vagrant boxes with 6GB of RAM (instead of 8), so that&apos;s an improvement. With JMX tooling in the Docker containers, we should be able to see easily whether GC is thrashing or fairly efficient.&lt;/p&gt;</comment>
                                                            <comment id="190962" author="5f9abc1eb45b2e007453f423" created="Wed, 20 Dec 2017 03:00:21 +0000"  >&lt;p&gt;Note that all Docker images and there startup scripts have been updated.  Use environment variable &apos;JAVA_OPTIONS&apos; to set JVM options in place of &apos;JAVA_OPTS&apos;.&lt;/p&gt;</comment>
                                                            <comment id="190965" author="5c706fbb47a54a6728e59df2" created="Fri, 30 Mar 2018 19:42:19 +0000"  >&lt;p&gt;Updated both testing and snapshot builds to include a &lt;tt&gt;-Xmx&lt;/tt&gt; JVM option for all Java-based containers. This buys us a little more space. We are still very crowded in an 8GB VM.&lt;/p&gt;</comment>
                                                            <comment id="190966" author="5c706fbb47a54a6728e59df2" created="Fri, 28 Sep 2018 22:21:31 +0000"  >&lt;p&gt;We likely need to build a more basic VM and make it easy for devs to deploy the modules they need for development, while still building the full-stack for the AWS demo boxes.&lt;/p&gt;</comment>
                                                            <comment id="190967" author="5c706fbb47a54a6728e59df2" created="Wed, 3 Oct 2018 15:23:15 +0000"  >&lt;p&gt;This issue can be closed &amp;#8211; we&apos;ve implemented what controls we can on container RAM usage in the VMs.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10000">
                    <name>Blocks</name>
                                            <outwardlinks description="blocks">
                                        <issuelink>
            <issuekey id="80874">FOLIO-1532</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="80783">FOLIO-1531</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="78559">MODVEND-12</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is blocked by">
                                        <issuelink>
            <issuekey id="80451">FOLIO-995</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                            <outwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="80441">FOLIO-977</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="80335">FOLIO-886</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="80426">FOLIO-974</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="64003" name="screenshot-1.png" size="125826" author="712020:32bb56ac-50e7-4787-b4af-ed3089d9401c" created="Thu, 7 Dec 2017 06:06:42 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10171"><![CDATA[Prokopovych]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|hzy4kv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 6 Dec 2017 20:13:10 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>