<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 23:21:39 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[FOLIO-2573] Paused due to POC of Elastic Search. All Search Options Should be Supported by a Database Index</title>
                <link>https://folio-org.atlassian.net/browse/FOLIO-2573</link>
                <project id="10290" key="FOLIO">FOLIO</project>
                    <description>&lt;p&gt;In order to (hopefully) pre-empt inadequate performance when searching inventory using the defined options in the reference UI, database indexes should be created for each CQL criteria that is generated by the reference UI&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Considerations&lt;/b&gt;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Should improve the baseline performance of inventory search (to be verified during the end of quarter release process)&lt;/li&gt;
	&lt;li&gt;Back end work required for every new search option e.g. a new filter&lt;/li&gt;
	&lt;li&gt;Will need to be maintained every time the UI changes it&apos;s query generation&lt;/li&gt;
	&lt;li&gt;May reduce write performance (especially when upgrading the system or migrating from existing systems)&lt;/li&gt;
	&lt;li&gt;May not improve performance in all cases&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="81714">FOLIO-2573</key>
            <summary>Paused due to POC of Elastic Search. All Search Options Should be Supported by a Database Index</summary>
                <type id="10003" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium">Task</type>
                                            <priority id="10002" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p3.svg">P3</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Won&apos;t Do</resolution>
                                                        <assignee accountid="-1">Unassigned</assignee>
                                                                <reporter accountid="63e2a2771b13d42998e4e706">Marc Johnson</reporter>
                                    <labels>
                            <label>PTF-Review</label>
                            <label>elastic-search</label>
                            <label>potential-decision</label>
                    </labels>
                <created>Fri, 24 Apr 2020 14:44:36 +0000</created>
                <updated>Mon, 31 Jan 2022 20:01:52 +0000</updated>
                            <resolved>Mon, 31 Jan 2022 20:01:52 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>22</watches>
                                                                <comments>
                                                            <comment id="198334" author="63e2a2771b13d42998e4e706" created="Fri, 24 Apr 2020 14:56:23 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5af5ed55244bc90a106063c7&quot; class=&quot;user-hover&quot; rel=&quot;5af5ed55244bc90a106063c7&quot; data-account-id=&quot;5af5ed55244bc90a106063c7&quot; accountid=&quot;5af5ed55244bc90a106063c7&quot; rel=&quot;noreferrer&quot;&gt;Cate Boerema&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Ab8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; class=&quot;user-hover&quot; rel=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; data-account-id=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; accountid=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; rel=&quot;noreferrer&quot;&gt;Jakub Skoczen&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=615afd1cd9820f0070a09ef0&quot; class=&quot;user-hover&quot; rel=&quot;615afd1cd9820f0070a09ef0&quot; data-account-id=&quot;615afd1cd9820f0070a09ef0&quot; accountid=&quot;615afd1cd9820f0070a09ef0&quot; rel=&quot;noreferrer&quot;&gt;Zak Burke&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Aff6a9612-bb35-41b2-88a8-a5b66d0a41a0&quot; class=&quot;user-hover&quot; rel=&quot;557058:ff6a9612-bb35-41b2-88a8-a5b66d0a41a0&quot; data-account-id=&quot;557058:ff6a9612-bb35-41b2-88a8-a5b66d0a41a0&quot; accountid=&quot;557058:ff6a9612-bb35-41b2-88a8-a5b66d0a41a0&quot; rel=&quot;noreferrer&quot;&gt;Michal Kuklis&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;There have been a number of ad-hoc conversations about this, I want to confirm that we are all in agreement that this is what we are going to do going forward?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Addc9bb7b-6444-4731-9688-566a04c9307c&quot; class=&quot;user-hover&quot; rel=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; data-account-id=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; accountid=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; rel=&quot;noreferrer&quot;&gt;Jeremy Huff&lt;/a&gt; &lt;/p&gt;

&lt;p&gt;Please raise any concerns you might have&lt;/p&gt;</comment>
                                                            <comment id="198336" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Fri, 24 Apr 2020 14:57:56 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt; - this looks really good &lt;img class=&quot;emoticon&quot; src=&quot;/images/icons/emoticons/star_yellow.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;

&lt;p&gt;CC: &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=70121%3Abe36acda-040d-4925-b3df-ebd7e16d4b5b&quot; class=&quot;user-hover&quot; rel=&quot;70121:be36acda-040d-4925-b3df-ebd7e16d4b5b&quot; data-account-id=&quot;70121:be36acda-040d-4925-b3df-ebd7e16d4b5b&quot; accountid=&quot;70121:be36acda-040d-4925-b3df-ebd7e16d4b5b&quot; rel=&quot;noreferrer&quot;&gt;lew235&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="198338" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 24 Apr 2020 15:03:20 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I think it is important to make an agreement what CQL query is going to be used for a particular story, because required indexes may depend on operators that are used in the CQL.&lt;/p&gt;</comment>
                                                            <comment id="198340" author="63e2a2771b13d42998e4e706" created="Fri, 24 Apr 2020 15:06:13 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;I think it is important to make an agreement what CQL query is going to be used for a particular story, because required indexes may depend on operators that are used in the CQL.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I totally agree, we are going to need to think about how we sequence the work to do that&lt;/p&gt;</comment>
                                                            <comment id="198341" author="5af5ed55244bc90a106063c7" created="Mon, 27 Apr 2020 08:01:30 +0000"  >&lt;blockquote&gt;&lt;p&gt;May reduce write performance (especially when upgrading the system or migrating from existing systems)&lt;br/&gt;
May not improve performance in all cases&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;These considerations concern me.  Rather than making this a blanket policy, is there any way to &lt;em&gt;only&lt;/em&gt; add indexes where they would improve search performance and not negatively impact write performance?&lt;/p&gt;

&lt;p&gt;I wonder if the folks on the performance testing task force have thoughts on this.  Tagging &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Ab45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; class=&quot;user-hover&quot; rel=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; data-account-id=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; accountid=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; rel=&quot;noreferrer&quot;&gt;Martin Tran&lt;/a&gt; for his thoughts.&lt;/p&gt;

&lt;p&gt;+ &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5d794c218ce6b60c3810eb8a&quot; class=&quot;user-hover&quot; rel=&quot;5d794c218ce6b60c3810eb8a&quot; data-account-id=&quot;5d794c218ce6b60c3810eb8a&quot; accountid=&quot;5d794c218ce6b60c3810eb8a&quot; rel=&quot;noreferrer&quot;&gt;Stephanie Buck&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="198345" author="63e2a2771b13d42998e4e706" created="Mon, 27 Apr 2020 16:12:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5af5ed55244bc90a106063c7&quot; class=&quot;user-hover&quot; rel=&quot;5af5ed55244bc90a106063c7&quot; data-account-id=&quot;5af5ed55244bc90a106063c7&quot; accountid=&quot;5af5ed55244bc90a106063c7&quot; rel=&quot;noreferrer&quot;&gt;Cate Boerema&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;is there any way to only add indexes where they would improve search performance and not negatively impact write performance?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;As I understand it, the addition of an index is likely to have a negative impact on write performance, as the index has to be updated when the write happens (in order for them to be consistent). To what degree that impact is, I don&apos;t know, it would require further investigation. It might be that after a while, the additional impact of each new index becomes close to zero.&lt;/p&gt;

&lt;p&gt;It&apos;s difficult to know for certain whether an index will increase performance for all searches. It is likely that each index will help significantly with a search that only contains the term it is designed to help. &lt;/p&gt;

&lt;p&gt;Once the search includes more than one term, it might even reduce the performance of a query because the database needs to decide which index to use. It might not always make the right decision. I believe this could be especially the case with the JSONB structures that FOLIO uses, as there aren&apos;t detailed statistics to help it make that decision, unlike regular columns.&lt;/p&gt;

&lt;p&gt;I think we would likely need performance tests for a wide variety of searches (and create / update scenarios) in order to learn more about the impact of these indexes.&lt;/p&gt;

&lt;p&gt;Does that make sense?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Ab8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; class=&quot;user-hover&quot; rel=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; data-account-id=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; accountid=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; rel=&quot;noreferrer&quot;&gt;Jakub Skoczen&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=70121%3A84bca0b8-4380-4d87-8a90-53e9133584f6&quot; class=&quot;user-hover&quot; rel=&quot;70121:84bca0b8-4380-4d87-8a90-53e9133584f6&quot; data-account-id=&quot;70121:84bca0b8-4380-4d87-8a90-53e9133584f6&quot; accountid=&quot;70121:84bca0b8-4380-4d87-8a90-53e9133584f6&quot; rel=&quot;noreferrer&quot;&gt;Mikhail Fokanov&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f8314dfbdef80006f6f572d&quot; class=&quot;user-hover&quot; rel=&quot;5f8314dfbdef80006f6f572d&quot; data-account-id=&quot;5f8314dfbdef80006f6f572d&quot; accountid=&quot;5f8314dfbdef80006f6f572d&quot; rel=&quot;noreferrer&quot;&gt;Adam Dickmeiss&lt;/a&gt; Please correct my understanding of any of the above.&lt;/p&gt;</comment>
                                                            <comment id="198347" author="5ee89462f7aa140abd82d11d" created="Wed, 6 May 2020 09:29:08 +0000"  >&lt;p&gt;Indexes reduce write performance, but this is very small and users cannot notice this write performance difference in the front-end.&lt;br/&gt;
You may notice the difference when doing bulk inserts or bulk updates, but this is small.&lt;br/&gt;
Some (not all) module upgrades require an index recreation, each index may take a few minutes depending on the number of records and the index type. Module upgrades are needed for each release, FOLIO publishes a new release every 3 months.&lt;br/&gt;
If an index speeds up the searches it should be used because the write performance penalty is very small.&lt;/p&gt;</comment>
                                                            <comment id="198350" author="63e2a2771b13d42998e4e706" created="Wed, 6 May 2020 10:38:40 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt; Thank you for your thoughts.&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;Indexes reduce write performance, but this is very small and users cannot notice this write performance difference in the front-end.&lt;br/&gt;
You may notice the difference when doing bulk inserts or bulk updates, but this is small.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Do we know what impact each index has?&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;Some (not all) module upgrades require an index recreation, each index may take a few minutes depending on the number of records and the index type.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I&apos;m thinking as much about upgrades that change many records. As I understand it, any change to a JSONB column involves a re-evaluation of all indexes based upon parts of that column. Is my understanding incorrect?&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;If an index speeds up the searches it should be used because the write performance penalty is very small.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Does that mean you are advocating for indexes that support every search in the reference inventory UI?&lt;/p&gt;

&lt;p&gt;Would you go further, and index all properties in inventory, as they can be used in the CQL search in the UI?&lt;/p&gt;</comment>
                                                            <comment id="198352" author="5ee89462f7aa140abd82d11d" created="Wed, 6 May 2020 17:14:12 +0000"  >&lt;blockquote&gt;&lt;p&gt;Do we know what impact each index has?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;It depends on average size of the field to be indexed and the number of records of the table.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;any change to a JSONB column involves a re-evaluation of all indexes based upon parts of that column.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;That is correct. Some initial PostgreSQL code for a recheck_on_update optimization exists, but it is not production ready. It calculates the index expression before and after the update and skips index update when they are the same.&lt;/p&gt;

&lt;p&gt;I advocate for creating indexes that give a noticeable speedup for all searches where we know that they will likely be executed.&lt;/p&gt;

&lt;p&gt;There will always be slow searches, for example when joining tables (instance, holding, item) and using indexed fields from instance and item.&lt;/p&gt;</comment>
                                                            <comment id="198354" author="63e2a2771b13d42998e4e706" created="Fri, 8 May 2020 11:31:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;It depends on average size of the field to be indexed and the number of records of the table.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Thanks, I wondered if we had a sense of the impact for a table with indexes from properties in a JSONB column with approximately 10 million records. I&apos;m not asking you to investigate this, I was only curious.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;I advocate for creating indexes that give a noticeable speedup for all searches where we know that they will likely be executed.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;How would you suggest we discover these indexes?&lt;/p&gt;

&lt;p&gt;Do you think it is reasonable to do this for all of the explicit search options in the reference UI (compound searches might still be slow)?&lt;/p&gt;

&lt;p&gt;A naive approximation, based upon 1 index per search option would mean the current reference UI would need a minimum of 15 indexes (some of which are custom like &lt;tt&gt;keyword&lt;/tt&gt;). More will be coming soon, including 4-9 (I think) for expansions to call number number searching.&lt;/p&gt;</comment>
                                                            <comment id="198356" author="5ee89462f7aa140abd82d11d" created="Fri, 8 May 2020 12:25:59 +0000"  >&lt;p&gt;Yes, it is reasonable to index all explicit search options in the UI.&lt;/p&gt;

&lt;p&gt;We should ask implementers to report slow queries. We may even create a slow queries page on wiki.folio.org where people can easily report them.&lt;/p&gt;</comment>
                                                            <comment id="198358" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Fri, 8 May 2020 12:32:11 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt; - it&apos;s pretty common to maintain search indexes for large bibliographic data bases with - e.g. I found this 57 p. document with spec for search codes and indexies in in DanBib. A Danish National Union Catalogue with all records formated as danMARC2 (a Danish variant of MARC21) - &lt;a href=&quot;https://slks.dk/fileadmin/user_upload/0_SLKS/Dokumenter/Biblioteker/Standarder/Praksisregler_for_soegeveje.pdf&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://slks.dk/fileadmin/user_upload/0_SLKS/Dokumenter/Biblioteker/Standarder/Praksisregler_for_soegeveje.pdf&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Have we an idea about, how we plan to list/document a list of implemented indexed searches?&lt;/p&gt;</comment>
                                                            <comment id="198360" author="63e2a2771b13d42998e4e706" created="Fri, 8 May 2020 12:44:06 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;Yes, it is reasonable to index all explicit search options in the UI.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Does that mean that that you are an advocate of creating (at least one, unless already present) index for each search option we add to the reference UI?&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;We should ask implementers to report slow queries. We may even create a slow queries page on wiki.folio.org where people can easily report them.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I think that is a good idea. I don&apos;t know if the FOLIO community has a process for accepting feedback from the implementers.&lt;/p&gt;</comment>
                                                            <comment id="198362" author="63e2a2771b13d42998e4e706" created="Fri, 8 May 2020 12:46:45 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;it&apos;s pretty common to maintain search indexes for large bibliographic data bases with - e.g. I found this 57 p. document with spec for search codes and indexies in in DanBib. A Danish National Union Catalogue with all records formated as danMARC2&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Ok. Does that mean you are advocating for FOLIO to create database indexes for each search option we add to the reference UI, based upon the fact that other systems do this?&lt;/p&gt;

&lt;p&gt;Unfortunately, that document does not help me understand the needs or the technical choices that organisation made because it is in a language I cannot read.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;Have we an idea about, how we plan to list/document a list of implemented indexed searches?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Not that I am aware of, to begin to answer that I&apos;d need to understand the need. &lt;/p&gt;

&lt;p&gt;What need / expectation would that documentation serve?&lt;/p&gt;</comment>
                                                            <comment id="198364" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Fri, 8 May 2020 13:09:38 +0000"  >&lt;p&gt;Sorry &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt; - I didn&apos;t intend that you should read that report in Danish. Just get the sense of how other systems do documentation of the search codes and indexes.&lt;/p&gt;

&lt;p&gt;Yes, I&apos;m advocating for that we in FOLIO do indexes for each of the search options and defined filters in the UI in Search and filter (first pane). And when adding a new search option or a filter, then we automatically should create the equivalent search index.&lt;/p&gt;

&lt;p&gt;These search indexes should be documented, e.g. as a table. &lt;/p&gt;

&lt;p&gt;This documentation could be transformed to a tips &amp;amp; trick document, for the staff user to know, what search indexes to use, when wanting to perform a given search. The MM-SIG has started following wiki page - &lt;a href=&quot;https://folio-org.atlassian.net/wiki/display/FOLIOtips/Searching&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://folio-org.atlassian.net/wiki/display/FOLIOtips/Searching&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="198366" author="63e2a2771b13d42998e4e706" created="Fri, 8 May 2020 13:20:49 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;I didn&apos;t intend that you should read that report in Danish. Just get the sense of how other systems do documentation of the search codes and indexes.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Thanks. It is challenging for me to get a sense of how other systems do this when I am unable to read the documentation itself, sorry :-/&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;Yes, I&apos;m advocating for that we in FOLIO do indexes for each of the search options and defined filters in the UI in the first page. And when adding a new search option or a filter, then we automatically should create the equivalent search index.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Ok. Alas, it isn&apos;t automatic and requires diligence and development work each time.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;These search indexes should be documented, e.g. as a table.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;This might sound that an impertinent question. Why do they need to be documented? What will this documentation be used for?&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;This documentation could be transformed to a tips &amp;amp; trick document, for the staff user to know, what search indexes to use, when wanting to perform a given search.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Does that mean this would be user facing documentation for them constructing their own queries outside of the ones provided by the reference UI?&lt;/p&gt;

&lt;p&gt;If so, I don&apos;t know what the process is for the creation of user facing documentation for FOLIO.&lt;/p&gt;</comment>
                                                            <comment id="198369" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Tue, 12 May 2020 14:15:23 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; class=&quot;user-hover&quot; rel=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; data-account-id=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; accountid=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; rel=&quot;noreferrer&quot;&gt;Theodor Tolstoy (One-Group.se)&lt;/a&gt; and &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5afc1ced2083b15a0bd3e494&quot; class=&quot;user-hover&quot; rel=&quot;5afc1ced2083b15a0bd3e494&quot; data-account-id=&quot;5afc1ced2083b15a0bd3e494&quot; accountid=&quot;5afc1ced2083b15a0bd3e494&quot; rel=&quot;noreferrer&quot;&gt;Lisa Sj&#246;gren&lt;/a&gt; - I have added you as watcher to this ticket, while I think the conversation in the #Support channel are addressing from a user perspective some of the challenges we&apos;re dealing with re. searches. &lt;/p&gt;

&lt;p&gt;Link to the original slack chat, thread 4/16/2020: in the #Support channel - &lt;a href=&quot;https://folio-project.slack.com/archives/C2BHMHCFJ/p1587026417102300&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://folio-project.slack.com/archives/C2BHMHCFJ/p1587026417102300&lt;/a&gt;&lt;br/&gt;
And to avoid risk loosing the thread, I also captured the conversation in a google doc: &lt;a href=&quot;https://docs.google.com/document/d/1AaAVpBiEtsK7cnaN3C57fudgmVodqNiIFpJraNncXEI/edit&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://docs.google.com/document/d/1AaAVpBiEtsK7cnaN3C57fudgmVodqNiIFpJraNncXEI/edit&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="198372" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Tue, 12 May 2020 15:22:19 +0000"  >&lt;p&gt;Comments/findings from &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;5/12/2020, 10:45 PM&amp;#93;&lt;/span&gt;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;OK, I have completed the db maintenance and brought the &lt;em&gt;&apos;large FOLIO Fameflower&apos;&lt;/em&gt; system back up. &lt;br/&gt;
Here&apos;s what I&apos;m finding in inventory search:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;SRS performance is only marginally better. It takes 8+ seconds to respond to a query like /source-storage/formattedRecords/3fedbd63-3e85-408e-a1b2-806cd8668802?identifier=INSTANCE&lt;/li&gt;
	&lt;li&gt;Filters that are backed by indexes (e.g. language, resource type, suppress from discovery) are performing well, except for mode of issuance.&lt;/li&gt;
	&lt;li&gt;Filters that do not have indexes (format, nature of content) create a large load on the database and do not perform, either timing out or taking a very long time.&lt;/li&gt;
	&lt;li&gt;Query search is Russian roulette. If there is an index for the property you are searching, it will work well. If not, performance will be unacceptable and DB CPU will spike&lt;/li&gt;
	&lt;li&gt;Holdings call number search is quite slow, which surprises me, as there should be an index for it.&lt;/li&gt;
&lt;/ol&gt;
&lt;/blockquote&gt;</comment>
                                                            <comment id="198374" author="712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2" created="Fri, 15 May 2020 09:00:22 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt; Thank you for pinging!&lt;br/&gt;
Re: Wayne&apos;s findings, I believe SRS has a different kind of problem than what we experience in Inventory. Hopefully, that will get fixed this quarter.&lt;/p&gt;

&lt;p&gt;. &lt;/p&gt;

</comment>
                                                            <comment id="198376" author="712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2" created="Fri, 15 May 2020 09:27:26 +0000"  >&lt;blockquote&gt;&lt;p&gt;Considerations&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;Should improve the baseline performance of inventory search (to be verified during the end of quarter release process)&lt;/li&gt;
	&lt;li&gt;Back end work required for every new search option e.g. a new filter&lt;/li&gt;
	&lt;li&gt;Will need to be maintained every time the UI changes it&apos;s query generation&lt;/li&gt;
	&lt;li&gt;May reduce write performance (especially when upgrading the system or migrating from existing systems)&lt;/li&gt;
	&lt;li&gt;May not improve performance in all cases&lt;/li&gt;
&lt;/ul&gt;
&lt;/blockquote&gt;

&lt;p&gt;I cannot help to think about how this list would have looked, had we had a proper search engine in place. &lt;/p&gt;</comment>
                                                            <comment id="198378" author="5afc1ced2083b15a0bd3e494" created="Fri, 15 May 2020 10:37:13 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt;! I am following these discussions around optimizing search performance (speed, usability, precision, internationalization), using either further indexing of fields or by adding a search engine, with great interest.&lt;/p&gt;

&lt;p&gt;And thank you &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=712020%3A0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; class=&quot;user-hover&quot; rel=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; data-account-id=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; accountid=&quot;712020:0a02d059-b8b9-48b3-8a58-727ec44d05d2&quot; rel=&quot;noreferrer&quot;&gt;Theodor Tolstoy (One-Group.se)&lt;/a&gt; for bringing up the search engine angle. I think it&apos;s important that the discussions around different approaches &amp;#8211; while they might both have their place, short and long term &amp;#8211; converge and inform each other.&lt;/p&gt;</comment>
                                                            <comment id="198383" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Fri, 15 May 2020 14:44:22 +0000"  >&lt;p&gt;Just a FYI: Link to a proposal (work in progress) published on the FOLIO wiki - &lt;a href=&quot;https://folio-org.atlassian.net/wiki/pages/viewpage.action?pageId=1776170&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://folio-org.atlassian.net/wiki/pages/viewpage.action?pageId=1776170&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="198385" author="63e2a2771b13d42998e4e706" created="Mon, 22 Jun 2020 14:48:23 +0000"  >&lt;p&gt;I think that the presence of issues reported (mostly by the EBSCO FSE or Performance Task Force teams) for searches that do not use a database index (e.g. 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;CIRCSTORE-215&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/CIRCSTORE-215&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;Missing Indexes Warnings &quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium&quot; /&gt;
            CIRCSTORE-215
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 or 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;MODINVSTOR-472&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/MODINVSTOR-472&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;Search queries without database index&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium&quot; /&gt;
            MODINVSTOR-472
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
) suggests to me that FOLIO is tending towards a policy of requiring a database index for every search query.&lt;/p&gt;

&lt;p&gt;Given that, I&apos;ve expanded the scope of this decision to be broader than inventory, and will add it to the topics for the Technical Leads to discuss.&lt;/p&gt;

&lt;p&gt;I think we need need to decide:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;whether a query that does not use a database index is considered invalid? (the corollary being, &lt;tt&gt;all CQL queries must execute using a database index&lt;/tt&gt;)&lt;/li&gt;
	&lt;li&gt;what should happen when a client request includes an invalid query?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;What do folks think? Do we need more information (a proposal) before this can be discussed further?&lt;/p&gt;</comment>
                                                            <comment id="198388" author="5ee89462f7aa140abd82d11d" created="Tue, 23 Jun 2020 16:02:08 +0000"  >&lt;p&gt;Small tables don&apos;t need database indexes because all queries are always fast, even when using a full table scan.&lt;/p&gt;</comment>
                                                            <comment id="198390" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Thu, 25 Jun 2020 09:43:48 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt; - could you give an example on, what you define as small tables? &lt;/p&gt;</comment>
                                                            <comment id="198392" author="5ee89462f7aa140abd82d11d" created="Thu, 25 Jun 2020 16:38:07 +0000"  >&lt;p&gt;10000 records. Most of these tables probably are small:&lt;br/&gt;
alternative_title_type&lt;br/&gt;
classification_type&lt;br/&gt;
contributor_name_type&lt;br/&gt;
contributor_type&lt;br/&gt;
identifier_type&lt;br/&gt;
instance_format&lt;br/&gt;
instance_relationship_type&lt;br/&gt;
instance_status&lt;br/&gt;
instance_type&lt;br/&gt;
loan_type&lt;br/&gt;
material_type&lt;br/&gt;
mode_of_issuance&lt;br/&gt;
nature_of_content_term&lt;br/&gt;
statistical_code_type&lt;/p&gt;</comment>
                                                            <comment id="198394" author="63e2a2771b13d42998e4e706" created="Fri, 3 Jul 2020 15:18:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Ab8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; class=&quot;user-hover&quot; rel=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; data-account-id=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; accountid=&quot;557058:b8e64633-1f7c-402d-9caf-9959a5ba5d0d&quot; rel=&quot;noreferrer&quot;&gt;Jakub Skoczen&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5cf6c546b87c300f36eb7b9a&quot; class=&quot;user-hover&quot; rel=&quot;5cf6c546b87c300f36eb7b9a&quot; data-account-id=&quot;5cf6c546b87c300f36eb7b9a&quot; accountid=&quot;5cf6c546b87c300f36eb7b9a&quot; rel=&quot;noreferrer&quot;&gt;Craig McNally&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5af5ed55244bc90a106063c7&quot; class=&quot;user-hover&quot; rel=&quot;5af5ed55244bc90a106063c7&quot; data-account-id=&quot;5af5ed55244bc90a106063c7&quot; accountid=&quot;5af5ed55244bc90a106063c7&quot; rel=&quot;noreferrer&quot;&gt;Cate Boerema&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Ab45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; class=&quot;user-hover&quot; rel=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; data-account-id=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; accountid=&quot;557058:b45447b0-4142-447e-9ac7-b2b97eeb90bc&quot; rel=&quot;noreferrer&quot;&gt;Martin Tran&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=70121%3A506eccfa-ceb4-4c24-b3be-f08c4364e2b9&quot; class=&quot;user-hover&quot; rel=&quot;70121:506eccfa-ceb4-4c24-b3be-f08c4364e2b9&quot; data-account-id=&quot;70121:506eccfa-ceb4-4c24-b3be-f08c4364e2b9&quot; accountid=&quot;70121:506eccfa-ceb4-4c24-b3be-f08c4364e2b9&quot; rel=&quot;noreferrer&quot;&gt;Svitlana Zmiivska&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Addc9bb7b-6444-4731-9688-566a04c9307c&quot; class=&quot;user-hover&quot; rel=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; data-account-id=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; accountid=&quot;557058:ddc9bb7b-6444-4731-9688-566a04c9307c&quot; rel=&quot;noreferrer&quot;&gt;Jeremy Huff&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt; &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=615afd1cd9820f0070a09ef0&quot; class=&quot;user-hover&quot; rel=&quot;615afd1cd9820f0070a09ef0&quot; data-account-id=&quot;615afd1cd9820f0070a09ef0&quot; accountid=&quot;615afd1cd9820f0070a09ef0&quot; rel=&quot;noreferrer&quot;&gt;Zak Burke&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Here is my current understanding of the situation&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;The Performance Task Force are observing the warning errors in the logs and raising issues to resolve them (see 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;CIRCSTORE-215&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/CIRCSTORE-215&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;Missing Indexes Warnings &quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium&quot; /&gt;
            CIRCSTORE-215
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 )&lt;/li&gt;
	&lt;li&gt;Long running queries are to be cancelled ( see 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;MODINVSTOR-496&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/MODINVSTOR-496&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;cancel long running instance searches&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10322?size=medium&quot; /&gt;
            MODINVSTOR-496
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 and 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;RMB-615&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/RMB-615&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;cancel queries that take longer than limit&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10322?size=medium&quot; /&gt;
            RMB-615
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 )&lt;/li&gt;
	&lt;li&gt;It has been proposed that all CQL queries that are not supported by a database index be disallowed ( see 
    &lt;span class=&quot;jira-issue-macro&quot; data-jira-key=&quot;RMB-534&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/RMB-534&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;Reject CQL queries that match no index in schema.json&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10322?size=medium&quot; /&gt;
            RMB-534
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-complete jira-macro-single-issue-export-pdf&quot;&gt;Open&lt;/span&gt;
            &lt;/span&gt;
 )&lt;/li&gt;
	&lt;li&gt;Folks want to be able to search on any property in inventory using CQL&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The top two of these effectively make a database index for many properties required.&lt;/p&gt;

&lt;p&gt;If we are to accommodate the fourth option then we either cannot allow the third to go ahead or all possible queries have to be catered for by an index. This would, in effect mean multiple indexes on every property in instances, holdings and items. &lt;/p&gt;

&lt;p&gt;Even if the impact of each index is relatively small, the cumulative effect of either of these could be significant.&lt;/p&gt;

&lt;p&gt;Have I understood the situation correctly?&lt;/p&gt;

&lt;p&gt;Is this the intended outcome of this combination of decisions?&lt;/p&gt;

&lt;p&gt;Should there be exceptions? As &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5ee89462f7aa140abd82d11d&quot; class=&quot;user-hover&quot; rel=&quot;5ee89462f7aa140abd82d11d&quot; data-account-id=&quot;5ee89462f7aa140abd82d11d&quot; accountid=&quot;5ee89462f7aa140abd82d11d&quot; rel=&quot;noreferrer&quot;&gt;Julian Ladisch&lt;/a&gt; correctly states, some record collections are small and so may not require indexes be sufficiently performant (I&apos;m deliberately not defining what that is in this post)&lt;/p&gt;

&lt;p&gt;Does a topic need raising with the Technical Leads or the Technical Council to turn this emerging defacto policy into actual policy?&lt;/p&gt;</comment>
                                                            <comment id="198532" author="63e2a2771b13d42998e4e706" created="Wed, 3 Feb 2021 12:06:02 +0000"  >&lt;p&gt;Blocking as is dependent upon the decisions made following the elastic search based search PoC&lt;/p&gt;</comment>
                                                            <comment id="198534" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Mon, 31 Jan 2022 18:28:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt; - can I close this ticket as &apos;Won&apos;t do&apos;, or do you want to keep it as an &apos;umbrealla&apos; ticket and keeping track  of the mod-search work?&lt;/p&gt;</comment>
                                                            <comment id="198536" author="63e2a2771b13d42998e4e706" created="Mon, 31 Jan 2022 18:43:29 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3Af80403de-e149-421e-8750-af45c853402f&quot; class=&quot;user-hover&quot; rel=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; data-account-id=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; accountid=&quot;557058:f80403de-e149-421e-8750-af45c853402f&quot; rel=&quot;noreferrer&quot;&gt;Charlotte Whitt&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;can I close this ticket as &apos;Won&apos;t do&apos;, or do you want to keep it as an &apos;umbrealla&apos; ticket and keeping track of the mod-search work?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I have no interest in this being kept as an umbrella for ES based search work.&lt;/p&gt;</comment>
                                                            <comment id="198538" author="557058:f80403de-e149-421e-8750-af45c853402f" created="Mon, 31 Jan 2022 20:01:52 +0000"  >&lt;p&gt;This work is now tracked in MSEARCH&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10000">
                    <name>Blocks</name>
                                                                <inwardlinks description="is blocked by">
                                        <issuelink>
            <issuekey id="11588">UXPROD-2806</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10008">
                    <name>Defines</name>
                                            <outwardlinks description="defines">
                                        <issuelink>
            <issuekey id="12376">UXPROD-3513</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10006">
                    <name>Gantt End to Start</name>
                                            <outwardlinks description="has to be done before">
                                        <issuelink>
            <issuekey id="48083">UIIN-790</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                            <outwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="26549">PERF-76</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56666">RMB-534</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56616">RMB-742</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="29989">CIRCSTORE-215</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56069">MODINVSTOR-496</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56075">MODINVSTOR-499</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56068">MODINVSTOR-510</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56100">MODINVSTOR-515</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="26752">PERF-48</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="57209">RMB-615</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56125">MODINVSTOR-472</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56071">MODINVSTOR-500</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="56143">MODINVSTOR-530</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10171"><![CDATA[Prokopovych]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|hzx3j1:er</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="852">Core: F - Sprint 87</customfieldvalue>
    <customfieldvalue id="824">Core: F - Sprint 88</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 24 Apr 2020 14:57:56 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>