<!-- 
RSS generated by JIRA (1001.0.0-SNAPSHOT#100246-sha1:7a5c50119eb0633d306e14180817ddef5e80c75d) at Fri Feb 09 00:07:27 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary add field=key&field=summary to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>FOLIO Jira</title>
    <link>https://folio-org.atlassian.net</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>1001.0.0-SNAPSHOT</version>
        <build-number>100246</build-number>
        <build-date>07-02-2024</build-date>
    </build-info>

<item>
            <title>[UXPROD-340] SPIKE: Prototype Kafka Message Queue</title>
                <link>https://folio-org.atlassian.net/browse/UXPROD-340</link>
                <project id="10000" key="UXPROD">UX Product</project>
                    <description>&lt;p&gt;Stand up an Apache Kafka instance for the purposes of receiving transaction data from Okapi.&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Clustered mode&lt;/li&gt;
	&lt;li&gt;Zookeeper&lt;/li&gt;
	&lt;li&gt;Access Control.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Provides &lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;a containerized (Docker) deliverable.&lt;/li&gt;
	&lt;li&gt;reference documentation&lt;/li&gt;
	&lt;li&gt;Pre-analysis of transaction data for optimization purposes&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="10245">UXPROD-340</key>
            <summary>SPIKE: Prototype Kafka Message Queue</summary>
                <type id="10005" iconUrl="https://folio-org.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10309?size=medium">Story</type>
                            <parent id="13686">UXPROD-330</parent>
                                    <priority id="10002" iconUrl="https://dev.folio.org/assets/jira-priority/jira-p3.svg">P3</priority>
                        <status id="6" iconUrl="https://folio-org.atlassian.net/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10003">Done</resolution>
                                                        <assignee accountid="70121:e70f578f-dc5a-4a30-8836-fc2418025d21">Matt Reno</assignee>
                                                                <reporter accountid="5cf6c265e7d2310e9fc0c5ac">VBar</reporter>
                                    <labels>
                            <label>analytics</label>
                            <label>kafka</label>
                    </labels>
                <created>Mon, 5 Mar 2018 05:04:26 +0000</created>
                <updated>Sat, 19 Jan 2019 00:22:06 +0000</updated>
                            <resolved>Mon, 15 Oct 2018 20:15:52 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>4</watches>
                                                                <comments>
                                                            <comment id="12700" author="557058:4f6bed01-40a6-48d5-8471-7ef21f5ea97c" created="Fri, 5 Oct 2018 15:52:27 +0000"  >&lt;p&gt;BTW, during earlier data capture POC, we used following to set up Kafaka&lt;br/&gt;
&lt;a href=&quot;https://hub.docker.com/r/wurstmeister/kafka/&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://hub.docker.com/r/wurstmeister/kafka/&lt;/a&gt;&lt;br/&gt;
&lt;a href=&quot;https://github.com/wurstmeister/kafka-docker&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/wurstmeister/kafka-docker&lt;/a&gt;&lt;/p&gt;</comment>
                                                            <comment id="12703" author="70121:e70f578f-dc5a-4a30-8836-fc2418025d21" created="Mon, 15 Oct 2018 12:58:04 +0000"  >&lt;p&gt;For this spike we installed a kafka cluster via a docker image on 3 EC2 instances. Additionally, a zookeeper ensemble was installed on these 3 instances.&lt;/p&gt;

&lt;p&gt;EC2 instances: t2.micro&lt;br/&gt;
	&#8226; Installed docker/docker-compose (instructions on the interwebs)&lt;br/&gt;
	&#8226; Installed git (instructions on the interwebs)&lt;/p&gt;

&lt;p&gt;Kafka: &lt;a href=&quot;https://github.com/wurstmeister/kafka-docker&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/wurstmeister/kafka-docker&lt;/a&gt;&lt;br/&gt;
	&#8226; git clone &lt;a href=&quot;https://github.com/wurstmeister/kafka-docker.git&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;https://github.com/wurstmeister/kafka-docker.git&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Edited docker-compose.yml file:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;ec2-user@ip-10-23-33-103 kafka-docker&amp;#93;&lt;/span&gt;$ cat docker-compose.yml&lt;/p&gt;

&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-none&quot;&gt;
version: &apos;2&apos;
services:
  zookeeper:
    image: 31z4/zookeeper
    restart: always
    hostname: zoo1
    ports:
      - 2181:2181
      - 3888:3888
      - 2888:2888
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=10.23.33.117:2888:3888 server.3=10.23.33.112:2888:3888
  kafka:
    build: .
    ports:
      - 9092:9092
    environment:
       KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_HEAP_OPTS: &quot;-Xmx256M -Xms128M&quot;
      KAFKA_ADVERTISED_HOST_NAME: 10.23.33.103
      KAFKA_ZOOKEEPER_CONNECT: 10.23.33.103:2181,10.23.33.117:2181,10.23.33.112:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;Other servers have different &quot;ZOO_MY_ID&quot; (incremented) values. The &quot;KAFKA_HEAP_OPTS&quot; setting is to allow for the small EC2 instance used to host the kafka docker image. By default it was trying to use 1GB, which caused an out of memory exception on startup. Also increased the replication  factor to 3 to provide some redundancy.&lt;/p&gt;

&lt;p&gt;mod-aes&lt;br/&gt;
	&#8226; Launched via AesVerticle with the following args:&lt;br/&gt;
		&#9675; -Dhttp.port=8081 -Dkafka.url=10.23.33.103:9092,10.23.33.117:9092,10.23.33.112:9092 -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory&lt;/p&gt;

&lt;p&gt;Curl was used to make a request to mod-aes:&lt;br/&gt;
	&#8226; $ curl -X POST   &lt;a href=&quot;http://localhost:8081/test&quot; class=&quot;external-link&quot; rel=&quot;nofollow noreferrer&quot;&gt;http://localhost:8081/test&lt;/a&gt; -H &apos;Content-Type: application/json&apos;   -H &apos;X-Okapi-Tenant: test&apos; -H &apos;x-okapi-filter: pre&apos;  -d &apos;&lt;/p&gt;
{&quot;test&quot;: &quot;some value&quot;}
&lt;p&gt;&apos;&lt;/p&gt;

&lt;p&gt;Next steps:&lt;br/&gt;
	&#8226; Securing kafka/zookeeper (to be another spike for a later sprint)&lt;br/&gt;
		&#9675; SSL connections&lt;br/&gt;
		&#9675; Kafka client authentication&lt;br/&gt;
		&#9675; Zookeeper client authentication&lt;br/&gt;
		&#9675; Kafka data-at-rest encryption&lt;br/&gt;
	&#8226; Cluster size optimization&lt;br/&gt;
		&#9675; Zookeeper and Kafka&lt;br/&gt;
	&#8226; Topic configuration (per tenant?)&lt;br/&gt;
		&#9675; Replication/partitioning/etc.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10003">
                    <name>Relates</name>
                                            <outwardlinks description="relates to">
                                        <issuelink>
            <issuekey id="10284">UXPROD-332</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="10817">UXPROD-1219</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Back End Estimate</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10121"><![CDATA[Medium < 5 days]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10054" key="com.atlassian.jira.plugin.system.customfieldtypes:userpicker">
                        <customfieldname>Back End Estimator</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5cf6c265e7d2310e9fc0c5ac</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10000" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummarycf">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10057" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Development Team</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10147"><![CDATA[EBSCO - FSE]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10014" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue key="$xmlutils.escape($text)">Reporting: Analytics and Audit Data Logging for External Reporting</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.plugin.system.customfieldtypes:userpicker">
                        <customfieldname>Front End Estimator</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5cf6c265e7d2310e9fc0c5ac</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10019" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|hzycif:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10020" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10024" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>[CHART] Date of First Response</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 5 Oct 2018 15:52:27 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10025" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>[CHART] Time in Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>