<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Thu Feb 08 23:25:38 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[FOLIO-3103] Restore mod-search/ui-inventory-es on ref envs</title>
                <link>https://folio-org.atlassian.net/browse/FOLIO-3103</link>
                <project id="10290" key="FOLIO">FOLIO</project>
                    <description>&lt;p&gt;The two modules was removed from snapshot deploy because of 
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;MSEARCH-86&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/MSEARCH-86&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;daily refenv build failed 20210402 mod-search-1.2.0-SNAPSHOT.57&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium&quot; /&gt;
            MSEARCH-86
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
.&lt;/p&gt;

&lt;p&gt;Once it is fixed we&apos;re able to restore the modules.&lt;/p&gt;</description>
                <environment></environment>
        <key id="82035">FOLIO-3103</key>
            <summary>Restore mod-search/ui-inventory-es on ref envs</summary>
                <type id="10003" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium">Task</type>
                                            <priority id="10001" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p2.svg">P2</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="5f9abc1eb45b2e007453f423">John Malconian</assignee>
                                                                <reporter accountid="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50">Bohdan Suprun</reporter>
                                    <labels>
                    </labels>
                <created>Mon, 5 Apr 2021 08:54:21 +0000</created>
                <updated>Fri, 23 Apr 2021 14:00:19 +0000</updated>
                            <resolved>Fri, 23 Apr 2021 14:00:19 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>4</watches>
                                                                <comments>
                                                            <comment id="196972" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Wed, 7 Apr 2021 14:14:58 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5cd423bebc70090d6ce241b1&quot; class=&quot;user-hover&quot; rel=&quot;5cd423bebc70090d6ce241b1&quot; data-account-id=&quot;5cd423bebc70090d6ce241b1&quot; accountid=&quot;5cd423bebc70090d6ce241b1&quot; rel=&quot;noreferrer&quot;&gt;Ian Hardy&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;
    &lt;span class=&quot;jira-issue-macro resolved&quot; data-jira-key=&quot;MSEARCH-86&quot; &gt;
                &lt;a href=&quot;https://folio-org.atlassian.net/browse/MSEARCH-86&quot; class=&quot;jira-issue-macro-key issue-link&quot;  title=&quot;daily refenv build failed 20210402 mod-search-1.2.0-SNAPSHOT.57&quot; &gt;
            &lt;img class=&quot;icon&quot; src=&quot;https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium&quot; /&gt;
            MSEARCH-86
        &lt;/a&gt;
                                                    &lt;span class=&quot;aui-lozenge aui-lozenge-subtle aui-lozenge-success jira-macro-single-issue-export-pdf&quot;&gt;Closed&lt;/span&gt;
            &lt;/span&gt;
 has been fixed, could you please add mod-search and ui-inventory-es to the ref envs? &lt;/p&gt;</comment>
                                                            <comment id="196976" author="5f9abc1eb45b2e007453f423" created="Thu, 8 Apr 2021 17:30:35 +0000"  >&lt;p&gt;We continue to see the same failure after trying the latest snapshot version of  mod-search committed to master (Build 66). &lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;fatal: [10.36.1.146]: FAILED! =&amp;gt; {&quot;changed&quot;: false, &quot;connection&quot;: &quot;close&quot;, &quot;content&quot;: &quot;POST request for mod-search-1.3.0-SNAPSHOT.66 /_/tenant failed with 500: {\&quot;errors\&quot;:[{\&quot;message\&quot;:\&quot;Failed to perform elasticsearch request [index=instance_diku, type=indexExists, message: elasticsearch: System error]\&quot;,\&quot;type\&quot;:\&quot;SearchOperationException\&quot;,\&quot;code\&quot;:\&quot;elasticsearch_error\&quot;}],\&quot;total_records\&quot;:1}&quot;, &quot;content_length&quot;: &quot;298&quot;, &quot;content_type&quot;: &quot;text/plain&quot;, &quot;elapsed&quot;: 160, &quot;msg&quot;: &quot;Status code was 400 and not [200]: HTTP Error 400: Bad Request&quot;, &quot;redirected&quot;: false, &quot;status&quot;: 400, &quot;url&quot;: &quot;http://10.36.1.146:9130/_/proxy/tenants/diku/install?deploy=true&amp;amp;tenantParameters=loadSample%3Dtrue%2CloadReference%3Dtrue&quot;, &quot;vary&quot;: &quot;origin&quot;}
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;Attaching complete Okapi logs to this issue. &lt;br/&gt;
Link to mod-search log: &lt;a href=&quot;https://folio-refenv-logs.s3.us-east-1.amazonaws.com/folio-snapshot-load/logs-2021-04-08-170426.tar.gz&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://folio-refenv-logs.s3.us-east-1.amazonaws.com/folio-snapshot-load/logs-2021-04-08-170426.tar.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I&apos;m afraid I will need to remove inventory-es and mod-search again from the snapshot builds. &lt;/p&gt;</comment>
                                                            <comment id="196981" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Thu, 8 Apr 2021 20:43:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Am I correct understand that this is deployment descriptor is being used? So the &lt;tt&gt;descriptor.env&lt;/tt&gt; is the list of ENV variables being passed to the container? &lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;changed: [10.36.1.146] =&amp;gt; (item={&apos;redirected&apos;: False, &apos;url&apos;: &apos;http://10.36.1.146:9130/_/discovery/modules/mod-search-1.3.0-SNAPSHOT.66&apos;, &apos;status&apos;: 404, &apos;vary&apos;: &apos;origin&apos;, &apos;content_type&apos;: &apos;text/plain&apos;, &apos;connection&apos;: &apos;close&apos;, &apos;content_length&apos;: &apos;28&apos;, &apos;msg&apos;: &apos;HTTP Error 404: Not Found&apos;, &apos;elapsed&apos;: 0, &apos;changed&apos;: False, &apos;invocation&apos;: {&apos;module_args&apos;: {&apos;url&apos;: &apos;http://10.36.1.146:9130/_/discovery/modules/mod-search-1.3.0-SNAPSHOT.66&apos;, &apos;headers&apos;: {&apos;X-Okapi-Tenant&apos;: &apos;supertenant&apos;, &apos;X-Okapi-Token&apos;: &apos;&apos;, &apos;Accept&apos;: &apos;application/json&apos;}, &apos;status_code&apos;: [&apos;200&apos;, &apos; 404&apos;], &apos;force&apos;: False, &apos;http_agent&apos;: &apos;ansible-httpget&apos;, &apos;use_proxy&apos;: True, &apos;validate_certs&apos;: True, &apos;force_basic_auth&apos;: False, &apos;body_format&apos;: &apos;raw&apos;, &apos;method&apos;: &apos;GET&apos;, &apos;return_content&apos;: False, &apos;follow_redirects&apos;: &apos;safe&apos;, &apos;timeout&apos;: 30, &apos;follow&apos;: False, &apos;url_username&apos;: None, &apos;url_password&apos;: None, &apos;client_cert&apos;: None, &apos;client_key&apos;: None, &apos;dest&apos;: None, &apos;body&apos;: None, &apos;src&apos;: None, &apos;creates&apos;: None, &apos;removes&apos;: None, &apos;unix_socket&apos;: None, &apos;mode&apos;: None, &apos;owner&apos;: None, &apos;group&apos;: None, &apos;seuser&apos;: None, &apos;serole&apos;: None, &apos;selevel&apos;: None, &apos;setype&apos;: None, &apos;attributes&apos;: None, &apos;content&apos;: None, &apos;backup&apos;: None, &apos;remote_src&apos;: None, &apos;regexp&apos;: None, &apos;delimiter&apos;: None, &apos;directory_mode&apos;: None, &apos;unsafe_writes&apos;: None}}, &apos;failed&apos;: False, &apos;item&apos;: {&apos;key&apos;: &apos;mod-search&apos;, &apos;value&apos;: {&apos;srvcId&apos;: &apos;mod-search-1.3.0-SNAPSHOT.66&apos;, &apos;nodeId&apos;: &apos;10.36.1.146&apos;, &apos;descriptor&apos;: {&apos;dockerImage&apos;: &apos;folioci/mod-search:1.3.0-SNAPSHOT.66&apos;, &apos;dockerPull&apos;: True, &apos;env&apos;: [{&apos;name&apos;: &apos;JAVA_OPTIONS&apos;, &apos;value&apos;: &apos;-XX:MaxRAMPercentage=66.0&apos;}, {&apos;name&apos;: &apos;KAFKA_HOST&apos;, &apos;value&apos;: &apos;kafka&apos;}, {&apos;name&apos;: &apos;KAFKA_PORT&apos;, &apos;value&apos;: &apos;9092&apos;}, {&apos;name&apos;: &apos;ELASTICSEARCH_HOST&apos;, &apos;value&apos;: &apos;elasticsearch&apos;}, {&apos;name&apos;: &apos;ELASTICSEARCH_PORT&apos;, &apos;value&apos;: &apos;9200&apos;}, {&apos;name&apos;: &apos;ELASTICSEARCH_URL&apos;, &apos;value&apos;: &apos;http://elasticsearch:9200&apos;}, {&apos;name&apos;: &apos;ELASTICSEARCH_USERNAME&apos;, &apos;value&apos;: &apos;elastic&apos;}, {&apos;name&apos;: &apos;ELASTICSEARCH_PASSWORD&apos;, &apos;value&apos;: &apos;s3cret&apos;}, {&apos;name&apos;: &apos;DB_HOST&apos;, &apos;value&apos;: &apos;postgres&apos;}, {&apos;name&apos;: &apos;DB_PORT&apos;, &apos;value&apos;: &apos;5432&apos;}, {&apos;name&apos;: &apos;DB_USERNAME&apos;, &apos;value&apos;: &apos;folio_admin&apos;}, {&apos;name&apos;: &apos;DB_PASSWORD&apos;, &apos;value&apos;: &apos;folio_admin&apos;}, {&apos;name&apos;: &apos;DB_DATABASE&apos;, &apos;value&apos;: &apos;okapi_modules&apos;}, {&apos;name&apos;: &apos;DB_QUERYTIMEOUT&apos;, &apos;value&apos;: &apos;60000&apos;}, {&apos;name&apos;: &apos;DB_CHARSET&apos;, &apos;value&apos;: &apos;UTF-8&apos;}, {&apos;name&apos;: &apos;DB_MAXPOOLSIZE&apos;, &apos;value&apos;: &apos;5&apos;}, {&apos;name&apos;: &apos;INITIAL_LANGUAGES&apos;, &apos;value&apos;: &apos;eng&apos;}, {&apos;name&apos;: &apos;SYSTEM_USER_PASSWORD&apos;, &apos;value&apos;: &apos;Mod-search-1-0-0&apos;}], &apos;dockerArgs&apos;: {&apos;HostConfig&apos;: {&apos;Memory&apos;: 536870912, &apos;PortBindings&apos;: {&apos;8081/tcp&apos;: [{&apos;HostPort&apos;: &apos;%p&apos;}]}}}}}}, &apos;ansible_loop_var&apos;: &apos;item&apos;})
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;If yes than could you make sure following:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Elasticsearch address is: &lt;tt&gt;&lt;a href=&quot;http://elasticsearch:9200&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://elasticsearch:9200&lt;/a&gt;&lt;/tt&gt;&lt;/li&gt;
	&lt;li&gt;Elasticsearch security is enabled and there is &lt;tt&gt;&lt;b&gt;elastic&lt;/b&gt;&lt;/tt&gt; user with &lt;tt&gt;&lt;b&gt;s3cret&lt;/b&gt;&lt;/tt&gt; password? If security is disabled than these variables must not be passed.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;The tenant init error suggests that mod-search event unable to connect to elasticsearch to check if index exists.&lt;/p&gt;</comment>
                                                            <comment id="196988" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 9 Apr 2021 11:27:32 +0000"  >&lt;p&gt;Ok, if my assumptions are correct and deployment tool copies env variables from module descriptor as they are, than the quickest fix is to leave only those variables that will be overridden by OKAPI: DB connection properties and ES host and port and remove all other properties.&lt;/p&gt;

&lt;p&gt;That&apos;s really surprising that we use &lt;b&gt;sample&lt;/b&gt; values for the variables during the &lt;b&gt;real&lt;/b&gt; deployment&#160;&lt;img class=&quot;emoticon&quot; src=&quot;/images/icons/emoticons/sad.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;.&lt;/p&gt;</comment>
                                                            <comment id="196992" author="5f9abc1eb45b2e007453f423" created="Fri, 9 Apr 2021 12:43:44 +0000"  >&lt;p&gt;I didn&apos;t set this up originally so let me investigate this.  thanks. &lt;/p&gt;</comment>
                                                            <comment id="196995" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Tue, 13 Apr 2021 11:38:17 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Have you had a chance to investigate it? &lt;/p&gt;

&lt;p&gt;We have &lt;a href=&quot;https://github.com/folio-org/mod-search/pull/70&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/mod-search/pull/70&lt;/a&gt; that should fix it, please review and approve if it looks fine.&lt;/p&gt;</comment>
                                                            <comment id="196999" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 03:15:18 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt; - I&apos;m still running into problems initializing the module for the tenant.    tenant initialization seems to result in an endless loop of log messages.   I&apos;ve attached a snippet of the log file (mod-search.log.gz)&lt;/p&gt;

&lt;p&gt;The following environment variables are passed to the module when the module is deployed (some are global): &lt;/p&gt;

&lt;p&gt;                &quot;KAFKA_HOST=10.36.1.77&quot;,&lt;br/&gt;
                &quot;ELASTICSEARCH_HOST=10.36.1.77&quot;,&lt;br/&gt;
                &quot;DB_MAXPOOLSIZE=5&quot;,&lt;br/&gt;
                &quot;DB_USERNAME=folio_admin&quot;,&lt;br/&gt;
                &quot;DB_PORT=5432&quot;,&lt;br/&gt;
                &quot;ELASTICSEARCH_URL=&lt;a href=&quot;http://10.36.1.77:9301&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://10.36.1.77:9301&lt;/a&gt;&quot;,&lt;br/&gt;
                &quot;DB_HOST=10.36.1.77&quot;,&lt;br/&gt;
                &quot;ELASTICSEARCH_PORT=9301&quot;,&lt;br/&gt;
                &quot;OKAPI_URL=&lt;a href=&quot;http://10.36.1.77:9130&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://10.36.1.77:9130&lt;/a&gt;&quot;,&lt;br/&gt;
                &quot;DB_DATABASE=okapi_modules&quot;,&lt;br/&gt;
                &quot;DB_PASSWORD=folio_admin5757&quot;,&lt;br/&gt;
                &quot;KAFKA_PORT=9092&quot;,&lt;br/&gt;
                &quot;PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin&quot;,&lt;br/&gt;
                &quot;LANG=en_US.UTF-8&quot;,&lt;br/&gt;
                &quot;LANGUAGE=en_US:en&quot;,&lt;br/&gt;
                &quot;LC_ALL=en_US.UTF-8&quot;,&lt;br/&gt;
                &quot;JAVA_VERSION=jdk-11.0.10+9&quot;,&lt;br/&gt;
                &quot;JAVA_HOME=/opt/java/openjdk&quot;,&lt;br/&gt;
                &quot;JAVA_APP_DIR=/opt/folio&quot;&lt;/p&gt;

&lt;p&gt;ELASTICSEARCH_USERNAME and ELASTICSEARCH_PASSWORD are not defined.  SYSTEM_USER_PASSWORD is not defined as well.   I also verified that &lt;a href=&quot;http://10.36.1.77:9301&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://10.36.1.77:9301&lt;/a&gt; (elasticsearch) is reachable.  &lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;
ubuntu@ip-10-36-1-77:~$ curl http://10.36.1.77:9301
{
  &quot;name&quot; : &quot;d4d862e05e63&quot;,
  &quot;cluster_name&quot; : &quot;docker-cluster&quot;,
  &quot;cluster_uuid&quot; : &quot;RrE6TcMkT3i4y9hV_HXFhg&quot;,
  &quot;version&quot; : {
    &quot;number&quot; : &quot;7.10.1&quot;,
    &quot;build_flavor&quot; : &quot;default&quot;,
    &quot;build_type&quot; : &quot;docker&quot;,
    &quot;build_hash&quot; : &quot;1c34507e66d7db1211f66f3513706fdf548736aa&quot;,
    &quot;build_date&quot; : &quot;2020-12-05T01:00:33.671820Z&quot;,
    &quot;build_snapshot&quot; : false,
    &quot;lucene_version&quot; : &quot;8.7.0&quot;,
    &quot;minimum_wire_compatibility_version&quot; : &quot;6.8.0&quot;,
    &quot;minimum_index_compatibility_version&quot; : &quot;6.0.0-beta1&quot;
  },
  &quot;tagline&quot; : &quot;You Know, for Search&quot;
}

&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;Any ideas? &lt;/p&gt;</comment>
                                                            <comment id="197003" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 06:40:55 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;That&apos;s something that expected if post tenant not invoked yet:&lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;org.folio.search.exception.TenantNotInitializedException: Following tenants might not be initialized yet: [diku]
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;After installing the module these errors should disappear.  &lt;/p&gt;</comment>
                                                            <comment id="197008" author="63e2a2771b13d42998e4e706" created="Fri, 16 Apr 2021 11:47:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;That&apos;s something that expected if post tenant not invoked yet&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;If it is expected and is not an issue, why is an exception thrown?&lt;/p&gt;
</comment>
                                                            <comment id="197012" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 12:15:05 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;What should happen in following case then:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Mod-inventory-storage started&lt;/li&gt;
	&lt;li&gt;A user sends a request to e.g. &lt;tt&gt;/instance-storage/instances&lt;/tt&gt; endpoint, &lt;b&gt;before&lt;/b&gt; POST /_/tenant is called&lt;/li&gt;
	&lt;li&gt;I&apos;m sure that the endpoint will return a 500 and an exception will be logged?&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;I don&apos;t think this is an issue and logging the exception is fine.&lt;/p&gt;</comment>
                                                            <comment id="197016" author="63e2a2771b13d42998e4e706" created="Fri, 16 Apr 2021 12:24:19 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Maybe I misunderstood the situation.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;What should happen in following case then&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I would consider your example to be somewhat exceptional as no HTTP requests should be made to a module directly only via Okapi and it knows which module versions are enabled for a given tenant, in which case this is either a bug in Okapi or a user is doing something that they shouldn&apos;t. Arguably, it shouldn&apos;t be an exception or a 500, rather a 400, as it could be considered a mistake on the part of the client to make a request to the module for a tenant that it is not responsible for or aware of.&lt;/p&gt;

&lt;p&gt;My understanding of the scenario that &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt; was describing is somewhat different. Does this error occur when a message is read from the log for a tenant that mod-search is not enabled for or that it is consuming from a tenant specific topic for a tenant it is not enabled for?&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;I don&apos;t think this is an issue and logging the exception is fine.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I was only trying to understand the reasoning. Apologies if that caused offence :-/ &lt;/p&gt;

&lt;p&gt;I get the sense that I shouldn&apos;t have asked the question. By all means, ignore it and I won&apos;t follow up on it further. Apologies :-/ &lt;/p&gt;</comment>
                                                            <comment id="197021" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 14:38:59 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=63e2a2771b13d42998e4e706&quot; class=&quot;user-hover&quot; rel=&quot;63e2a2771b13d42998e4e706&quot; data-account-id=&quot;63e2a2771b13d42998e4e706&quot; accountid=&quot;63e2a2771b13d42998e4e706&quot; rel=&quot;noreferrer&quot;&gt;Marc Johnson&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;No problem at all.&lt;/p&gt;

&lt;p&gt;We don&apos;t have separate topics for tenants for now. Issue occurs when mod-search tries to consume messages for tenant for which it is not enabled yet. The error is not handled, just to let Kafka send message again so that we&apos;re guaranteeing that it is handled when the tenant is enabled (or another temporary issue is resolved - e.g. if inventory unavailable for a moment - we use &lt;tt&gt;/inventory-view/instances&lt;/tt&gt; API to get latest representation of instance/item/holding).&lt;/p&gt;

&lt;p&gt;Of course we can handle the exception but in this case some messages will be lost when a temporary failure occurred.   &lt;/p&gt;

&lt;p&gt;We also had a spike to make it a little bit clearer, and now if we recognize that tenant is not initialized yet we wrap the cause with a &lt;tt&gt;TenantNotInitializedException&lt;/tt&gt;.&lt;/p&gt;</comment>
                                                            <comment id="197026" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 15:18:14 +0000"  >&lt;p&gt;For record keeping:   mod-search deployed with the following options. &lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;ok: [10.36.1.41] =&amp;gt; (item={&apos;key&apos;: &apos;mod-search&apos;, &apos;value&apos;: {&apos;srvcId&apos;: &apos;mod-search-1.3.0-SNAPSHOT.72&apos;, &apos;nodeId&apos;: &apos;10.36.1.41&apos;, &apos;descriptor&apos;: {&apos;dockerImage&apos;: &apos;folioci/mod-search:1.3.0-SNAPSHOT.72&apos;, &apos;dockerPull&apos;: True, &apos;env&apos;: [{&apos;name&apos;: &apos;ELASTICSEARCH_URL&apos;, &apos;value&apos;: &apos;http://10.36.1.41:9301&apos;}, {&apos;name&apos;: &apos;OKAPI_URL&apos;, &apos;value&apos;: &apos;http://10.36.1.41:9130&apos;}, {&apos;name&apos;: &apos;JAVA_OPTIONS&apos;, &apos;value&apos;: &apos;-XX:MaxRAMPercentage=66.0&apos;}], &apos;dockerArgs&apos;: {&apos;HostConfig&apos;: {&apos;Memory&apos;: 536870912, &apos;PortBindings&apos;: {&apos;8081/tcp&apos;: [{&apos;HostPort&apos;: &apos;%p&apos;}]}}}}}})

&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                                                            <comment id="197029" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 16:54:50 +0000"  >&lt;p&gt;There is quite bit of resource consumption going on with mod-search while it waits for the tenant initialization and the &quot;system_user&quot;. &lt;/p&gt;


&lt;p&gt;CONTAINER ID   NAME           CPU %     MEM USAGE / LIMIT   MEM %     NET I/O          BLOCK I/O   PIDS&lt;br/&gt;
ed6dea71c888   clever_ellis   95.45%    381.3MiB / 512MiB   74.48%    10.8GB / 228MB   0B / 0B     43&lt;/p&gt;</comment>
                                                            <comment id="197033" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 17:46:13 +0000"  >&lt;p&gt;So far in my testing of FOLIO reference builds,  I&apos;m getting a mix of build failures and successes when mod-search is included.  When successful, an additional 10 minutes is added to the total tenant initialization time.   Right now,  my sample size is small,  so I will conduct more tests.  &lt;/p&gt;</comment>
                                                            <comment id="197036" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 18:47:46 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;. Is there any CPU limitation for mod-search?&lt;/p&gt;

&lt;p&gt;Could you please provide whole logs for mod-search when it failed next time? I want to check how long it takes to init the app, to call tenant init etc.&lt;/p&gt;</comment>
                                                            <comment id="197040" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 19:40:41 +0000"  >&lt;p&gt;I guess there is something with this OKAPI orchestration mode. What I mentioned in logs that there is significant delay between when module initialized fully and when tenant init is called. &lt;/p&gt;

&lt;p&gt;For example MSEARCH-83: &lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;# Application started
00:10:39.884 [main] INFO  org.folio.search.SearchApplication - Started SearchApplication in 9.25 seconds (JVM running for 10.249)

# Tenant init called
00:29:19.092 [http-nio-8081-exec-2] INFO  org.folio.spring.controller.TenantController - Initializing tenant...
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;As you can see it is 19 mins delay. I&apos;ll have to investigate OKAPI code, OKAPI should should have a way when it is assumes that module is initialized and then sends the post tenant. There should be an issue here, I think. &lt;/p&gt;</comment>
                                                            <comment id="197043" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 19:50:04 +0000"  >&lt;p&gt;thanks &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt; - I had to pull inventory-es and mod-search out of snapshot again since I was getting more build failures than successful ones.    There is a delay between module deployment and tenant initialization.   First,  Okapi deploys all backend modules which takes a certain amount of time and then we enable all backend modules at once for the tenant using the Okapi install endpoint.  It is this tenant initialization process that is giving us troubles.  Normally this step takes less than 5 minutes, but with mod-search included, it can take an additional 10 minutes (if successful). Otherwise, the it seems to abort.    One thing that we also need to check is the connection timeout between the Ansible client and Okapi as well as the ALB and Okapi.   When failures occur,  there is no error from Okapi.  It appears that Ansible is just aborting the connection.   Regardless of a possible connection timeout,  we need to determine what is causing the extra time to initialize the module with the tenant. &lt;/p&gt;</comment>
                                                            <comment id="197048" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Fri, 16 Apr 2021 20:09:50 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;, &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5c706fbb47a54a6728e59df2&quot; class=&quot;user-hover&quot; rel=&quot;5c706fbb47a54a6728e59df2&quot; data-account-id=&quot;5c706fbb47a54a6728e59df2&quot; accountid=&quot;5c706fbb47a54a6728e59df2&quot; rel=&quot;noreferrer&quot;&gt;Wayne Schneider&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Is it possible to run playbook inside a vagrant box so I can debug it? &lt;/p&gt;</comment>
                                                            <comment id="197050" author="5f9abc1eb45b2e007453f423" created="Tue, 20 Apr 2021 16:34:31 +0000"  >&lt;p&gt;@bohdan - I think on the vagrant box,  the best option would be to create a second tenant and initialize the same set of modules for the second tenant that the default diku tenant is running + mod-search to get an idea of what&apos;s going on.  &lt;/p&gt;

&lt;p&gt;But another issue has come up that is preventing me from doing further troubleshooting.  ui-inventory-es needs a dependency update.  &lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;Incompatible version for module folio_inventory-es-6.2.100047 interface circulation. Need 9.0. Have 10.0/mod-circulation-21.0.0-SNAPSHOT.836
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;I&apos;m also wondering if it&apos;s possible to back off the polling loop in mod-search to every 10-15 seconds.   I&apos;m not sure what it is now,  but it&apos;s generating a ton of noise in the logs.  &lt;/p&gt;</comment>
                                                            <comment id="197055" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Wed, 21 Apr 2021 08:53:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Sure, there is already a PR - &lt;a href=&quot;https://github.com/folio-org/mod-search/pull/74&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/mod-search/pull/74&lt;/a&gt; to increase back-off to 20 seconds.&lt;/p&gt;

&lt;p&gt;And I also created a PR to update versions &lt;a href=&quot;https://github.com/folio-org/ui-inventory-es/pull/44&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/ui-inventory-es/pull/44&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="197058" author="5f9abc1eb45b2e007453f423" created="Wed, 21 Apr 2021 15:18:37 +0000"  >&lt;p&gt;Excellent,  thanks &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt;.   Ping me when &lt;a href=&quot;https://github.com/folio-org/mod-search/pull/74&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/mod-search/pull/74&lt;/a&gt; has been merged.   &lt;/p&gt;</comment>
                                                            <comment id="197063" author="557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50" created="Thu, 22 Apr 2021 13:05:31 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=5f9abc1eb45b2e007453f423&quot; class=&quot;user-hover&quot; rel=&quot;5f9abc1eb45b2e007453f423&quot; data-account-id=&quot;5f9abc1eb45b2e007453f423&quot; accountid=&quot;5f9abc1eb45b2e007453f423&quot; rel=&quot;noreferrer&quot;&gt;John Malconian&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;We have merged the &lt;a href=&quot;https://github.com/folio-org/mod-search/pull/74&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/folio-org/mod-search/pull/74&lt;/a&gt; and one more &lt;a href=&quot;https://github.com/folio-org/mod-search/pull/75&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;MSEARCH-112: Check that ES index exists before any indexing logic.&lt;/a&gt;&#160;improvement that prevents module to call OKAPI before ES indexes are created (i.e. before module is initialized).&lt;/p&gt;</comment>
                                                            <comment id="197066" author="5f9abc1eb45b2e007453f423" created="Thu, 22 Apr 2021 18:20:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://folio-org.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; class=&quot;user-hover&quot; rel=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; data-account-id=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; accountid=&quot;557058:89dc6567-baed-429c-bebb-dfcc7e5dcc50&quot; rel=&quot;noreferrer&quot;&gt;Bohdan Suprun&lt;/a&gt; The recent changes made to logging were excellent.   Before the changes,  the module was generating ~10gb of I/O.   After the change, ~2MB.    Tenant initialization time after the changes is now relatively the same as before.    I will re-add inventory-es/mod-search to snapshot and then close this issue.  &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10000">
                    <name>Blocks</name>
                                            <outwardlinks description="blocks">
                                        <issuelink>
            <issuekey id="82029">FOLIO-3091</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is blocked by">
                                        <issuelink>
            <issuekey id="28754">MSEARCH-86</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="64381" name="mod-search.log.gz" size="3259" author="5f9abc1eb45b2e007453f423" created="Fri, 16 Apr 2021 03:15:49 +0000"/>
                            <attachment id="64382" name="okapi-search.tar.gz" size="3221409" author="5f9abc1eb45b2e007453f423" created="Thu, 8 Apr 2021 17:29:03 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10155"><![CDATA[FOLIO DevOps]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10063" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>PO Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i02gli:9</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="955">DevOps Sprint 111</customfieldvalue>
    <customfieldvalue id="956">DevOps Sprint 112</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 8 Apr 2021 17:30:35 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>