<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 23:19:24 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[FOLIO-2264] mod-agreements crashes, out of memory</title>
                <link>https://folio-org.atlassian.net/browse/FOLIO-2264</link>
                <project id="10290" key="FOLIO">FOLIO</project>
                    <description>&lt;p&gt;mod-agreements crashes shorlty after it&apos;s enabled due to out of memory error. Make sure agreements is getting enough memory, and no bumping into the container&apos;s memory limit.&lt;/p&gt;</description>
                <environment></environment>
        <key id="81398">FOLIO-2264</key>
            <summary>mod-agreements crashes, out of memory</summary>
                <type id="10003" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium">Task</type>
                                            <priority id="10005" iconUrl="https://dev.folio.org/assets/jira-priority/tbd.svg">TBD</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="5cd423bebc70090d6ce241b1">Ian Hardy</assignee>
                                                                <reporter accountid="5cd423bebc70090d6ce241b1">Ian Hardy</reporter>
                                    <labels>
                            <label>ci</label>
                            <label>devops</label>
                            <label>platform-backlog</label>
                    </labels>
                <created>Tue, 17 Sep 2019 14:28:09 +0000</created>
                <updated>Wed, 3 Jun 2020 16:40:08 +0000</updated>
                            <resolved>Fri, 20 Sep 2019 12:08:21 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>6</watches>
                                                                <comments>
                                                            <comment id="193155" author="5cd423bebc70090d6ce241b1" created="Wed, 18 Sep 2019 14:19:12 +0000"  >&lt;p&gt;After a conversation w &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5bfd62c75f77da6929583d6c&quot; class=&quot;user-hover&quot; rel=&quot;5bfd62c75f77da6929583d6c&quot; data-account-id=&quot;5bfd62c75f77da6929583d6c&quot; accountid=&quot;5bfd62c75f77da6929583d6c&quot; rel=&quot;noreferrer&quot;&gt;steve.osguthorpe&lt;/a&gt; this could be caused by bumping into the system wide memory limit during tenant init. Steve will try limiting memory usage in mod-agreements as a test. &lt;/p&gt;</comment>
                                                            <comment id="193158" author="557058:28cce430-5ccd-47e7-88fe-1b26c20ee795" created="Wed, 18 Sep 2019 15:19:19 +0000"  >&lt;p&gt;Outstanding questions:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;8 containers were killed - were they all agreements? If so, how were they being &lt;span class=&quot;error&quot;&gt;&amp;#91;re&amp;#93;&lt;/span&gt;started&lt;/li&gt;
	&lt;li&gt;The version of mod-agreements&lt;/li&gt;
	&lt;li&gt;Number of processor cores&lt;/li&gt;
	&lt;li&gt;Total system ram for the machine (not the container)&lt;/li&gt;
	&lt;li&gt;Container process startup params (i.e. number of cpus,memory etc) if limited&lt;/li&gt;
	&lt;li&gt;JVM startup parameters (xmx and xms especially)&lt;br/&gt;
Can you also let me know if you are running the full suite of folio modules (i.e. all ~50 containers)?&lt;br/&gt;
Does the container work as expected if restarted without any other change?&lt;/li&gt;
&lt;/ul&gt;
</comment>
                                                            <comment id="193163" author="5d8a51e1bcc5480dd6ddfec7" created="Wed, 18 Sep 2019 16:11:32 +0000"  >&lt;p&gt;To answer a lot of questions we use AWS ECS to maintain a desired number of containers.  This desired value is 1 for mod-agreements.  All of the containers that I scanned through were mod-agreements containers that had last logged entries during the time of our issues.  The module was failing health checks and attempted to restart several times.  That is my interpretation of the 8 different containers .  &lt;/p&gt;

&lt;p&gt;We identified that mod-agreements requires 1028 soft limit and 1280 hard limit to run without issue.  We implemented these values recently after our issues yesterday morning so we are still tracking the behavior.   &lt;/p&gt;

&lt;p&gt;mod-agreements-1.9.0 &lt;br/&gt;
m5.large 2 vCpu and 8GiB memory&lt;br/&gt;
128 cpu units allocated to mod-agreements, 512 soft and not hard limited at time of the failure.&lt;br/&gt;
we do not add any JVM args to the modules currently&lt;/p&gt;

&lt;p&gt;we do not have all 50 modules running on this instance but they are running in the larger deployment.  There are approximately 15 containers on the instance including mod-agreements.&lt;/p&gt;

&lt;p&gt;Hopefully that answers your questions, let me know if you need more info&lt;/p&gt;
</comment>
                                                            <comment id="193169" author="5bfd62c75f77da6929583d6c" created="Wed, 18 Sep 2019 17:47:37 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5d8a51e1bcc5480dd6ddfec7&quot; class=&quot;user-hover&quot; rel=&quot;5d8a51e1bcc5480dd6ddfec7&quot; data-account-id=&quot;5d8a51e1bcc5480dd6ddfec7&quot; accountid=&quot;5d8a51e1bcc5480dd6ddfec7&quot; rel=&quot;noreferrer&quot;&gt;Eric Valuk&lt;/a&gt; Thanks for this. And I think we have got to the bottom of this and it&apos;s to do with metaspace usage by the JVM. This is memory that java processes will use directly and is not part of the heap space allocated by the &apos;Xm(s/x)&apos; parameters, and is instead written directly to the system memory. By default this is &quot;unbounded&quot; unless specified explicitly when starting the JVM process. I am going to do some more testing to come up with sensible values for mod-agreements, but this could actually affect any module not explicitly limiting the metaspace for the JVM.&lt;/p&gt;

&lt;p&gt;For now I have reduced thread counts in mod-agreements and also specified some hard limits for buffers, as well as moving the buffers out of metaspace and into heap. We have also limited the JVM on the index-data controlled reference environments to 256M of ram and this has allowed things to operate in the default ~700M container size. These changes obviously have a detrimental effect on the app though in terms of performance so I will be changing these to be more introspective of the resourcing available.&lt;/p&gt;</comment>
                                                            <comment id="193176" author="5cd423bebc70090d6ce241b1" created="Wed, 18 Sep 2019 17:55:41 +0000"  >&lt;p&gt;In our environments (32 gb ram, 8 cpu running all 50 mods. Head size set to 384 for mod-agreements) we were seeing just mod-agreements get killed when a host memory limit for the container was set. Steve&apos;s made some changes to the latest snapshot of mod-agreements to set limits which in combination with lowering the -Xmx value to 256 in folio-ansible seem to give agreements enough headspace for now.&lt;/p&gt;</comment>
                                                            <comment id="193182" author="5cd423bebc70090d6ce241b1" created="Thu, 19 Sep 2019 19:18:32 +0000"  >&lt;p&gt;Also effects mod-licenses. Steve&apos;s recommended new defaults for the JAVA_OPTIONS. Will verify builds tomorrow AM.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="64256" name="agreements-inspect.json" size="6752" author="5cd423bebc70090d6ce241b1" created="Tue, 17 Sep 2019 14:28:39 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10144"><![CDATA[Core: Platform]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i0052n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1469">CP: sprint 72</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10044" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>Story Points</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 18 Sep 2019 15:19:19 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>