Page Comparison

Table of Contents

...

Some lists can’t be created at all, and even if they are, there's a high chance they’ll fail when trying to refresh.
Memory utilization increasing for mod-fgm-manager up to 80% during lists refresh then goes down in few minutes - OOM issue for mod-fqm-manager
Jira Legacy
server System Jira
serverId 01505d01-b853-3c2e-90f1-ee9b165564fc
key MODFQMMGR-501
but only for creating new lists while the test query is running.
For mod-lists, CPU utilization was up to 237% for the test with 10 concurrent users and R/W split enabled. Average CPU usage did not exceed 14 % for all other modules.
Approximately DB CPU usage is up to 98.5%.

...

10 m6g.2xlarge EC2 instances located in US East (N. Virginia)us-east-1
2 instances of db.r6g.xlarge database, writer instance, and reader instance.
MSK - fse-tenant
4 kafka.m7g.xlarge brokers in 2 zones
- Apache Kafka version 3.7.x
- EBS storage volume per broker 300 GiB
- auto.create.topics.enable=true
- log.retention.minutes=480
- default.replication.factor=3
OpenSearch ptf-testcluster
- OpenSearch version 2.13
- Data nodes
  - Availability Zone(s) - 2-AZ without standby
  - Instance type - r6g.2xlarge.search
  - Number of nodes - 4
  - EBS volume size (GiB) - 300
  - Provisioned IOPS - 3000IOPS
  - Provisioned Throughput (MiB/s) - 250 MiB/s

Module versions

mod-fqm-manager -R/W split enabled

Cluster Resources - rcp1
Module	Task Definition Revision	Module Version	Task Count	Mem Hard Limit	Mem Soft Limit	CPU Units	Xmx	Metaspace Size	Max Metaspace Size
mod-authtoken	1	mod-authtoken:2.16.0-SNAPSHOT.303	2	1440	1152	512	922	88	128
mod-configuration	1	mod-configuration:5.11.0-SNAPSHOT.355	2	1024	896	0	768	88	128
mod-orders-storage	1	mod-orders-storage:13.8.0-SNAPSHOT.243	2	1024	896	512	700	88	128
mod-fqm-manager	10	mod-fqm-manager:2.1.0-SNAPSHOT.1197	2	3000	2600	128	2048	384	512
mod-finance	1	mod-finance:5.0.0-SNAPSHOT.174	2	1024	896	0	700	88	128
mod-lists	5	mod-lists:2.1.0-SNAPSHOT.1303	2	1024	896	0	768	88	128
mod-entities-links	4	mod-entities-links:3.1.0-SNAPSHOT.113	2	2592	2480	400	1440	0	1024
mod-permissions	2	mod-permissions:6.6.0-SNAPSHOT.385	2	1684	1544	512	1024	384	512
mod-orders	1	mod-orders:12.9.0-SNAPSHOT.398	2	2048	1440	1024	1024	384	512
edge-patron	1	edge-patron:5.2.0-SNAPSHOT.174	2	1024	896	256	768	88	128
mod-invoice	1	mod-invoice:5.9.0-SNAPSHOT.205	2	1440	1152	512	922	88	128
mod-inventory-storage	4	mod-inventory-storage:27.2.0-SNAPSHOT.773	2	4096	3690	2048	3076	512	1024
mod-circulation-storage	1	mod-circulation-storage:17.3.0-SNAPSHOT.405	2	2880	2592	1536	1814	384	512
mod-source-record-storage	2	mod-source-record-storage:5.9.0-SNAPSHOT.405	2	5600	5000	2048	3500	384	512
mod-inventory	1	mod-inventory:20.3.0-SNAPSHOT.546	2	2880	2592	1024	1814	384	512
mod-users	1	mod-users:19.3.2-SNAPSHOT.696	2	1024	896	0	768	88	128
edge-fqm	1	edge-fqm:2.1.0-SNAPSHOT.100	2	1024	896	0	768	88	128
mod-source-record-manager	1	mod-source-record-manager:3.9.0-SNAPSHOT.353	2	5600	5000	2048	3500	384	512
nginx-edge	1	nginx-edge:2023.06.14	2	1024	896	0	0	0	0
nginx-okapi	1	nginx-okapi:2023.06.14	2	1024	896	0	0	0	0
okapi-b	1	okapi:5.3.0	3	1684	1440	1024	922	384	512
mod-invoice-storage	1	mod-invoice-storage:5.9.0-SNAPSHOT.138	2	1872	1536	1024	1024	384	512
mod-service-interaction	1	mod-service-interaction:4.1.0-SNAPSHOT.126	2	2048	1844	256	1290	384	512
mod-login	1	mod-login:7.12.0-SNAPSHOT.324	2	1440	1298	1024	768	384	512
mod-organizations-storage	1	mod-organizations-storage:4.8.0-SNAPSHOT.131	2	1024	896	0	700	88	128
pub-okapi	1	pub-okapi:2023.06.14	2	1024	896	0	768	0	0

fqm-manager rivision

Expand

title	mod-fqm-manager, rivision #10

{
"taskDefinitionArn": "arn:aws:ecs:us-east-1:054267740449:task-definition/rcp1-mod-fqm-manager:10",
"containerDefinitions": [
{
"name": "mod-fqm-manager",
"image": "579891902283.dkr.ecr.us-east-1.amazonaws.com/folio/mod-fqm-manager:2.1.0-SNAPSHOT.1197",
"cpu": 128,
"memory": 3000,
"memoryReservation": 2600,
"portMappings": [
{
"containerPort": 8081,
"hostPort": 0,
"protocol": "tcp"
}
],
"essential": true,
"environment": [
{
"name": "mod-fqm-manager.query-retention-duration",
"value": "24h"
},
{
"name": "DB_MAXPOOLSIZE",
"value": "20"
},
{
"name": "CONFIG_FILE",
"value": "config.json"
},
{
"name": "JAVA_PROFILER_STATE",
"value": "disabled"
},
{
"name": "AWS_REGION",
"value": "us-east-1"
},
{
"name": "MAX_QUERY_SIZE",
"value": "30000000"
},
{
"name": "task.execution.pool.max-size",
"value": "10"
},
{
"name": "DB_PORT",
"value": "5432"
},
{
"name": "task.execution.pool.core-size",
"value": "9"
},
{
"name": "JAVA_ARGS",
"value": "--server.port=8082 -Dlog.level=info"
},
{
"name": "JAVA_OPTS",
"value": "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/usr/ms/mod-fqm-manager.hprof -XX:OnOutOfMemoryError=/usr/ms/heapdump.sh -XX:MetaspaceSize=384m -XX:MaxMetaspaceSize=512m -Xmx2048m"
},
{
"name": "ENV",
"value": "rcp1"
},
{
"name": "DB_HOST",
"value": "db.rcp1.folio-eis.us-east-1"
},
{
"name": "DB_HOST_READER",
"value": "db-ro.rcp1.folio-eis.us-east-1"
},
{
"name": "JAVA_PROFILER_OPTS",
"value": "-noverify -javaagent:\"/usr/ms/jvm-profiler-1.0.0.jar\"=configProvider=com.uber.profiling.YamlConfigProvider,configFile=\"/usr/ms/profiler.yaml\" -cp \"/usr/ms/jvm-profiler-1.0.0.jar\" "
},
{
"name": "LOG4J_CONFIGURATION_FILE",
"value": "https://s3.amazonaws.com/rcp1-folio-eis-us-east-1-int/log/log4j2.properties"
},
{
"name": "DB_PORT_READER",
"value": "5432"
},
{
"name": "PREFIX",
"value": "rcp1"
},
{
"name": "DB_DATABASE",
"value": "folio"
},
{
"name": "DB_EXPLAIN_QUERY_THRESHOLD",
"value": "300000"
},
{
"name": "task.execution.pool.queue-capacity",
"value": "1000"
}
],
"mountPoints": [],
"volumesFrom": [],
"secrets": [
{
"name": "DB_USERNAME",
"valueFrom": "arn:aws:ssm:us-east-1:054267740449:parameter/fse/cluster/rcp1/dbClusterMaster_userName"
},
{
"name": "DB_PASSWORD",
"valueFrom": "arn:aws:ssm:us-east-1:054267740449:parameter/fse/cluster/rcp1/dbClusterMaster_userPassword"
}
],
"stopTimeout": 120,
"ulimits": [
{
"name": "nofile",
"softLimit": 1048576,
"hardLimit": 1048576
}
],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "rcp1-folio-eis",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "rcp1"
}
},
"systemControls": []
}
],
"family": "rcp1-mod-fqm-manager",
"taskRoleArn": "arn:aws:iam::054267740449:role/Role-folio-ecs-task",
"executionRoleArn": "arn:aws:iam::054267740449:role/Role-folio-ecs-task",
"revision": 10,
"volumes": [],
"status": "ACTIVE",
"requiresAttributes": [
{
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"name": "ecs.capability.execution-role-awslogs"
},
{
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"name": "ecs.capability.container-ordering"
},
{
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"name": "ecs.capability.secrets.ssm.environment-variables"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
}
],
"placementConstraints": [],
"compatibilities": [
"EXTERNAL",
"EC2"
],
"registeredAt": "2024-10-04T09:27:29.988Z",
"registeredBy": "arn:aws:sts::054267740449:assumed-role/AWSReservedSSO_FOLIOFSE_ead3c38ca817a601/okondratenko@ebsco.com",
"enableFaultInjection": false,
"tags": []
}

Methodology/Approach

To test Baseline for List refresh JMeter scripts were used.

...

Versions Compared

Old Version 13

New Version 14

Key

Module versions

Methodology/Approach