<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Fri Feb 09 00:30:13 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[UXPROD-3210] NFR: R1 2022 Lotus Data import Stability/Reliability work </title>
                <link>https://folio-org.atlassian.net/browse/UXPROD-3210</link>
                <project id="10000" key="UXPROD">UX Product</project>
                    <description>&lt;p&gt;&lt;font color=&quot;red&quot;&gt;BE est is Jumbo; rough estimate is 120 days&lt;/font&gt;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Current situation or problem&lt;/b&gt;:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;High CPU/Memory consumption on modules&lt;/li&gt;
	&lt;li&gt;Duplicates may created upon import for holdings and items (instances were fixed)&lt;/li&gt;
	&lt;li&gt;Confirm that SRS does fail when processing during import&lt;br/&gt;
#&#160;If we have infrastructure issue (like DB not available, module being restarted or network failure), we are sending DI_ERROR instead of retrying&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;b&gt;Investigation required for:&lt;/b&gt;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Race condition on start (Kafka consumers start working before DB is configured) OR Periodical DB shutdown after SRS restart. Jobs get stuck if not able to update status in DB (messages ACKed even if we could not process them)&lt;/li&gt;
	&lt;li&gt;Kafka consumers stop reading messages eventually, breaking job progress until module restart.&lt;/li&gt;
	&lt;li&gt;mod-data-import stores input file in memory, limiting size of uploaded file and possibly having oom&lt;/li&gt;
	&lt;li&gt;Consumer gets disconnected from Kafka cluster&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&lt;b&gt;Proposed solution/stories&lt;/b&gt;&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Make consumers behave idempotent. Add pass-through identifier to de-duplicate messages.&lt;/li&gt;
	&lt;li&gt;Generate &quot;INSTANCE CREATED&quot; from mod-inventory. Consume in SRS to update HRID in BIB and in INVENTORY to continue processing.&lt;/li&gt;
	&lt;li&gt;Do not ACK messages in Kafka if there&apos;s not a logic, but infrastructure error/exception. Split failed processing results into 2 categories:
	&lt;ul&gt;
		&lt;li&gt;IO errors - do not ack. retry until fixed&lt;/li&gt;
		&lt;li&gt;Business logic - DI_ERROR and Ack current message&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;Remove unnecessary topics (* ready for post processing and hrid set)&lt;/li&gt;
	&lt;li&gt;De-duplicate status messages per-record while tracking progress&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;One possible solution: Split to chunks, put to database, work with database/temp storage. Partially done&#160;(to be investigated)&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Links to additional info:&lt;/b&gt;&lt;br/&gt;
&lt;b&gt;Update to wherever the plan is now stored&lt;/b&gt;&lt;br/&gt;
 &lt;b&gt;&lt;a href=&quot;https://wiki.folio.org/pages/resumedraft.action?draftId=65115261&amp;amp;draftShareId=5061165d-96d0-4c00-a9f9-3a18b80833b2&amp;amp;&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;Data Import Stabilization plan - Vladimir Shalaev - FOLIO Wiki&lt;/a&gt;&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Questions&lt;/b&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="12087">UXPROD-3210</key>
            <summary>NFR: R1 2022 Lotus Data import Stability/Reliability work </summary>
                <type id="10002" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10322?size=medium">New Feature</type>
                            <parent id="13571">UXPROD-47</parent>
                                    <priority id="10001" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p2.svg">P2</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="5af5e627525ba96b58654f12">Ann-Marie Breaux</assignee>
                                                                <reporter accountid="60a5273475e875006f2e05a0">Taisiya Trunova</reporter>
                                    <labels>
                            <label>NFR</label>
                    </labels>
                <created>Thu, 12 Aug 2021 08:39:21 +0000</created>
                <updated>Mon, 28 Feb 2022 15:48:51 +0000</updated>
                            <resolved>Mon, 28 Feb 2022 15:48:51 +0000</resolved>
                                                    <fixVersion>Lotus (R1 2022)</fixVersion>
                                        <due></due>
                            <votes>0</votes>
                                    <watches>2</watches>
                                                                    <issuelinks>
                            <issuelinktype id="10010">
                    <name>Continues</name>
                                            <outwardlinks description="continues">
                                        <issuelink>
            <issuekey id="12140">UXPROD-3193</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is continued by">
                                        <issuelink>
            <issuekey id="12195">UXPROD-3429</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10008">
                    <name>Defines</name>
                                                                <inwardlinks description="is defined by ">
                                        <issuelink>
            <issuekey id="39455">KAFKAWRAP-7</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62565">MODDATAIMP-473</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62628">MODDATAIMP-495</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62645">MODDATAIMP-500</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60399">MODINV-408</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="63227">MODSOURCE-402</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="58091">MODSOURMAN-598</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="39453">KAFKAWRAP-3</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62624">MODDATAIMP-491</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62652">MODDATAIMP-566</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60546">MODINV-460</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60469">MODINV-547</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60471">MODINV-548</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60573">MODINV-584</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60660">MODINV-588</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60662">MODINV-589</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60664">MODINV-590</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="60666">MODINV-591</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="66568">MODINVOICE-252</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="63210">MODSOURCE-290</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="58004">MODSOURMAN-474</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                                                <inwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="12291">UXPROD-3471</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Back End Estimate</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10126"><![CDATA[Jumbo: > 45 days]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10153"><![CDATA[Folijet]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10014" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue key="$xmlutils.escape($text)">Batch Importer (Bib/Acq)</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10050" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Front End Estimate</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10094"><![CDATA[Out of scope]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10052" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Front-End Confidence factor</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10107"><![CDATA[Low]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10063" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>PO Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>125.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i036fz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10069" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Rank: Cornell (Full Sum 2021)</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10213"><![CDATA[R1]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10091" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Rank: U of AL (MVP Oct 2020)</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10324"><![CDATA[R1]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10046" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Release</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10052"><![CDATA[Lotus R1 2022]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>