<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 22:31:54 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[MODMARCMIG-7] Implement Async Records Mapping Mechanism for MARC Migration (Process Chunks)</title>
                <link>https://folio-org.atlassian.net/browse/MODMARCMIG-7</link>
                <project id="10268" key="MODMARCMIG">mod-marc-migrations</project>
                    <description>&lt;p&gt;&lt;b&gt;Overview:&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;Prior to commencing the asynchronous data mapping for segmented chunks as part of the MARC migration, this story focuses on setting up the groundwork necessary for data mapping. It involves creating a table to manage chunk steps and preparing the infrastructure for parallel processing of chunks to facilitate efficient data mapping.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Requirements/Scope:&lt;/b&gt;&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Establish a database table specifically for managing chunk steps during the data mapping process.&lt;/li&gt;
	&lt;li&gt;Initiate data mapping processing once the operation status changes to &quot;DATA_MAPPING.&quot;&lt;/li&gt;
	&lt;li&gt;For each chunk, perform data mapping in parallel, preparing and storing necessary files while handling any potential errors.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;b&gt;Approach:&lt;/b&gt;&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;&lt;b&gt;Create Table for Chunk Step Management:&lt;/b&gt;
	&lt;ul&gt;
		&lt;li&gt;Develop a database table structure named &quot;chunk_step&quot; to monitor and manage the progress of chunk-based data mapping.&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;&lt;b&gt;Trigger Parallel Chunk Processing on Status Change:&lt;/b&gt;
	&lt;ul&gt;
		&lt;li&gt;Start the data mapping process for chunks in parallel once the status transitions to &quot;DATA_MAPPING&quot; for the registered MARC migration operation.&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;&lt;b&gt;Process Individual Chunks:&lt;/b&gt;
	&lt;ul&gt;
		&lt;li&gt;Prepare local files for the authority entities, invalid MARC records, and error records for each chunk.&lt;/li&gt;
		&lt;li&gt;Stream records from the database and perform mapping operations for each record.&lt;/li&gt;
		&lt;li&gt;Save successfully mapped records to the entity file, store problematic records in the invalid records file, and log error causes in the error records file.&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;&lt;b&gt;Store Files in Persistent Storage and Update Tables:&lt;/b&gt;
	&lt;ul&gt;
		&lt;li&gt;Persist the generated files (authority entities, invalid records, error records) in the designated persistent storage.&lt;/li&gt;
		&lt;li&gt;Register the file names and associated details (chunk ID, file paths) in both the chunk and chunk_step tables for tracking purposes.&lt;/li&gt;
		&lt;li&gt;Update status of operation to DATA_MAPPING_COMPLETED or DATA_MAPPING_FAILED&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;b&gt;Additional info:&lt;/b&gt;&lt;br/&gt;
POC for defining chunks and mapping: &lt;a href=&quot;https://github.com/folio-org/mod-entities-links/commits/MODELINKS-84-long-term-poc/&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/mod-entities-links/commits/MODELINKS-84-long-term-poc/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;S3-like client lib: &lt;a href=&quot;https://github.com/folio-org/folio-s3-client&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/folio-s3-client&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Acceptance Criteria:&lt;/b&gt;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Covered by Unit tests&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="79274">MODMARCMIG-7</key>
            <summary>Implement Async Records Mapping Mechanism for MARC Migration (Process Chunks)</summary>
                <type id="10005" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium">Story</type>
                            <parent id="79273">MODMARCMIG-10</parent>
                                    <priority id="10002" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p3.svg">P3</priority>
                        <status id="3" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/inprogress.png" description="This issue is being actively worked on at the moment by the assignee.">In Progress</status>
                    <statusCategory id="4" key="indeterminate" colorName="yellow"/>
                                    <resolution id="-1">Unresolved</resolution>
                                                        <assignee accountid="712020:79f3369a-5acf-46b0-ba9b-37a802c63526">Viacheslav Kolesnyk</assignee>
                                                                <reporter accountid="5eb44b5a96bbcb0b8585c98d">Pavlo Smahin</reporter>
                                    <labels>
                            <label>back-end</label>
                            <label>epam-spitfire</label>
                    </labels>
                <created>Tue, 12 Dec 2023 11:11:00 +0000</created>
                <updated>Wed, 7 Feb 2024 16:23:07 +0000</updated>
                                                            <fixVersion>1.0.0</fixVersion>
                                        <due></due>
                            <votes>0</votes>
                                    <watches>1</watches>
                                                                    <issuelinks>
                            <issuelinktype id="10006">
                    <name>Gantt End to Start</name>
                                            <outwardlinks description="has to be done before">
                                        <issuelink>
            <issuekey id="79267">MODMARCMIG-9</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="has to be done after">
                                        <issuelink>
            <issuekey id="79269">MODMARCMIG-6</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                            <outwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="13764">UXPROD-4082</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10180"><![CDATA[Spitfire]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10014" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue key="$xmlutils.escape($text)">Authority remapping</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10063" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>PO Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i098vy:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10046" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Release</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10079"><![CDATA[Quesnelia (R1 2024)]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="969">Spitfire Sprint 184</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10044" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>Story Points</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            </customfields>
    </item>
</channel>
</rss>