<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:cc="http://cyber.law.harvard.edu/rss/creativeCommonsRssModule.html">
    <channel>
        <title><![CDATA[Stories by Ankit Juneja on Medium]]></title>
        <description><![CDATA[Stories by Ankit Juneja on Medium]]></description>
        <link>https://medium.com/@ankitjuneja?source=rss-31751420d0e2------2</link>
        <image>
            <url>https://cdn-images-1.medium.com/fit/c/150/150/0*vKBEmB4EAcfvQxAE.</url>
            <title>Stories by Ankit Juneja on Medium</title>
            <link>https://medium.com/@ankitjuneja?source=rss-31751420d0e2------2</link>
        </image>
        <generator>Medium</generator>
        <lastBuildDate>Sat, 21 Nov 2026 21:25:13 GMT</lastBuildDate>
        <atom:link href="https://medium.com/@ankitjuneja/feed" rel="self" type="application/rss+xml"/>
        <webMaster><![CDATA[yourfriends@medium.com]]></webMaster>
        <atom:link href="http://medium.superfeedr.com/" rel="hub"/>
        <item>
            <title><![CDATA[Sap Backup -3]]></title>
            <link>https://medium.com/@ankitjuneja/sap-backup-3-412f129e92fd?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/412f129e92fd</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Fri, 22 Aug 2025 04:07:39 GMT</pubDate>
            <atom:updated>2025-08-22T04:07:39.435Z</atom:updated>
            <content:encoded><![CDATA[<p>Collation</p><pre>collation<br><br><br>collation_test<br><br><br><br><br><br><br>collation<br>resources<br>avro<br>Reservationgroupblockmerge<br>irrecoverable<br>recoverable<br><br><br>connection.config<br><br><br>dbconfig<br>  {<br>    jdbcconfig =<br>      {<br>        default<br>          {<br>            jdbcurl = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-rds-proxy.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>            drivername = &quot;org.postgresql.Driver&quot;<br>            username = &quot;au_spark_stream_app&quot;<br>            password = &quot;*************&quot;<br>            poolsize = 10<br>            idletimeout = 1200000<br>            querytimeout=&quot;50000 millis&quot;<br>            jdbcurlro = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-endpoint.endpoint.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>            drivernamero = &quot;org.postgresql.Driver&quot;<br>            usernamero = &quot;au_spark_stream_app_ro&quot;<br>            passwordro = &quot;*************&quot;<br>            poolsizero = 10<br>            idletimeoutro = 1200000<br>            querytimeoutro=&quot;50000 millis&quot;<br>            jdbcurlwo = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-rds-proxy.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>            drivernamewo = &quot;org.postgresql.Driver&quot;<br>            usernamewo = &quot;au_spark_stream_app&quot;<br>            passwordwo = &quot;*************&quot;<br>            poolsizewo = 10<br>            idletimeoutwo = 1200000<br>            querytimeoutwo=&quot;50000 millis&quot;<br>          }<br>        sgds<br>          {<br>            jdbcurl = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-sgds-rds-proxy.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>            drivername = &quot;org.postgresql.Driver&quot;<br>            username = &quot;au_spark_stream_app&quot;<br>            password = &quot;*************&quot;<br>            poolsize = 10<br>            idletimeout = 1200000<br>            querytimeout=&quot;50000 millis&quot;<br>            jdbcurlro = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-sgds-endpoint.endpoint.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>            drivernamero = &quot;org.postgresql.Driver&quot;<br>            usernamero = &quot;au_spark_stream_app_ro&quot;<br>            passwordro = &quot;*************&quot;<br>            poolsizero = 10<br>            idletimeoutro = 1200000<br>            querytimeoutro=&quot;50000 millis&quot;<br>            jdbcurlwo = &quot;jdbc:postgresql://rms-oy-dev-aurora-postgres-sgds-rds-proxy.proxy-cbgemarietsq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>            drivernamewo = &quot;org.postgresql.Driver&quot;<br>            usernamewo = &quot;au_spark_stream_app&quot;<br>            passwordwo = &quot;*************&quot;<br>            poolsizewo = 10<br>            idletimeoutwo = 1200000<br>            querytimeoutwo=&quot;50000 millis&quot;<br>          }<br>      }<br>  }<br><br>brokerconfig {<br><br>  kafkaconfig {<br><br>    default {<br>      // bootstrapserver = &quot;172.25.26.146:9092,172.25.26.243:9092,172.25.27.51:9092&quot; // no SSL<br>      bootstrapserver = &quot;http://platform1-devtest1-aws-us-east-1.aws-use1.cloud.marriott.com:8081&quot;<br>      kafkakeyserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;<br>      kafkavalueserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;<br>      //kafkasecurityprotocol = &quot;SSL&quot;<br>      //kafkassltruststorelocation = &quot;rms-emrkafka-consumer-dev-truststore.jks&quot;<br>      //kafkasslkeystorelocation = &quot;rms-emrkafka-consumer-dev-keystore.jks&quot;<br>      //kafkassltruststorepassword = &quot;test123&quot;<br>      //kafkasslkeystorepassword = &quot;test123&quot;<br>      //kafkaidentificationalgorithm = &quot;&quot;<br>      kafkatruststoretype = &quot;JKS&quot;<br>    }<br>  }<br>}<br><br>bucketconfig {<br>  bucket = {<br>    default = &quot;s3://rms-oyty-dev-emr/GroupBlockDetails/checkpointinglocation/&quot;<br>  }<br>}<br><br>schemaconfig {<br>  schemamap = {<br>    SchemaName = &quot;ymtest&quot;<br>  }<br>}<br><br>serviceconfig {<br>  servicemap = {<br>    ApplicationRunIdServiceUrl = &quot;&quot;&quot;http://appdata.oysclouddev.marriott.com/rms/application/v1/properties/propCodeKey/runid&quot;&quot;&quot;<br>    ApplicationFeatureServiceUrl = &quot;&quot;&quot;http://appdata.oyscloudtst.marriott.com/rms/application/v1/properties/propCodeKey/featurestatus&quot;&quot;&quot;<br>    ORAnalyticsServiceUrl = &quot;&quot;&quot;http://oy-refresh-forecast-calc-rms-ordev1.ose-dev39-red.aws-use1.cloud.marriott.com/ngicapi/1.0/forecast/refresh/json&quot;&quot;&quot;<br>    BenchMarkRatesServiceUrl = &quot;&quot;&quot;http://rateods.oyscloudtst.marriott.com/rms/rateods/v1/properties/propCodeKey/benchmarkrates&quot;&quot;&quot;<br>    HurdleRevenueServiceUrl = &quot;&quot;&quot;http://inventorycontrol.oyscloudtst.marriott.com/rms/inventorycontrol/v1/properties/propCodeKey/stays/hurdles&quot;&quot;&quot;<br>    RateForecastPredictServiceUrl = &quot;&quot;&quot;http://172.25.31.28:9302/predict/&quot;&quot;&quot;<br>    OptimizationWindowServiceUrl = &quot;&quot;&quot;http://172.25.31.28:8085/implement/window/propCodeKey/start&quot;&quot;&quot;<br>    CoreImplementsServiceUrl = &quot;&quot;&quot;http://172.25.31.28:8085/implement/opt/propCodeKey/start&quot;&quot;&quot;<br>    KeyStoreName = &quot;dev-keystore.jks&quot;<br>    KeyStorePassword = &quot;*************&quot;<br>    master=&quot;local&quot;<br>  }<br><br>}<br><br>jobconfig.config<br><br><br>jobconfigurationdetails {<br>  jobidentifier = GroupBlock<br>  jobname = GroupBlockCollation<br>  harddeletelist = []<br>  triggerinterval = 2000<br>  partitionvalue = 4<br>  checkpointdirectory {<br>    checkpointDir = &quot;checkpointDir&quot;<br>    IrrecoverableErrorCheckPoint=&quot;IrrecoverableErrorCheckPoint&quot;<br>    GroupBlockCollationDBCheckpointDir=&quot;checkpointinglocation/GroupBlockCollationDBCheckpointDir&quot;<br>    GroupBlockCollationKafkaCheckpointDir=&quot;checkpointinglocation/GroupBlockCollationKafkaCheckpointDir&quot;<br>    GroupBlockCollationRecoverableEntityCheckpointDir=&quot;checkpointinglocation/GroupBlockCollationRecoverableEntityCheckpointDir&quot;<br>    GroupBlockCollationIrrecoverableEntityCheckpointDir=&quot;checkpointinglocation/GroupBlockCollationIrrecoverableEntityCheckpointDir&quot;<br>  }<br><br>  dependencymap {}<br><br>  kafkatopicmap {<br>    SourceTopic = &quot;commerce.reservation.group-block.sync-test&quot;<br>    DestinationTopic = &quot;revenue-management.yield-management.group-block.merge-test&quot;<br>    DestinationTopicFilePath = &quot;ReservationGroupBlockMerge0.1.avsc&quot;<br>    RecoverableErrorTopic = &quot;commerce.reservation.group-block.collation.recoverable-test.dlt&quot;<br>    RecoverableFilePath = &quot;recoverable-DLQ.avsc&quot;<br>    IrrecoverableErrorTopic = &quot;commerce.reservation.group-block.collation.irrecoverable-test.dlt&quot;<br>    IrrecoverableFilePath = &quot;irrecoverable-DLQ.avsc&quot;<br>    MaxOffsetsPerTrigger = &quot;500&quot;<br>  }<br><br>  tablelookupconfig = [{<br>    srctable = &quot;pj_gb&quot;<br>    targettable = &quot;sgds_quote&quot;<br>    srccols = [&quot;ROW_ID&quot;]<br>    targetcols = [&quot;qo_quote_id&quot;]<br>  }]<br><br>  schemamap {<br>    TGTGroupBlockTable = &quot;pj_gb&quot;<br>    PropertyTableIN = &quot;property&quot;<br>    UserControlTableIN = &quot;user_control&quot;<br>    PropParamAssignTableIN = &quot;prop_param_assign&quot;<br>  }<br>  sqlmap {<br>    propertyquery = &quot;&quot;&quot;select pt_prop_code, pt_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(PropertyTable).get) + &quot; where pt_prop_code=&quot; + &quot;&#39;&quot; + propCode + &quot;&#39;&quot; + &quot; and pt_brand_code in (select uc_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(UserControlTable).get) + &quot; where uc_brand_oy_cmnty=&#39;Y&#39; and uc_prop_code=&#39;ALL00&#39;)&quot;&quot;&quot;<br>    PpaPropCodeQuery = s&quot;&quot;&quot;select $PpaPropCode  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTable).get} where $PpaParamId=&#39;DATASRCIND&#39; and $PpaParamValue=&#39;S&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;&quot;&quot;<br>    PpaParamQuery = s&quot;&quot;&quot;select $PpaParamValue  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTable).get} where $PpaParamId=&#39;SELLCURCD&#39; and $PpaActiveInd=&#39;Y&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;&quot;&quot;<br>  }<br>}<br><br>kafka.properties<br><br>spring.kafka.listener.ack-mode=${KAFKA_LISTENER_ACK_MODE:MANUAL_IMMEDIATE}<br>app.kafka.isConsumerEnabled=${KAFKA_CONSUMER_ENABLED:true}<br>app.kafka.isProducerEnabled=${KAFKA_PRODUCER_ENABLED:true}<br>app.kafka.sslConfig=${SSL_CONFIG:SASL_SSL}<br><br>app.kafka.consumerKeyStorePath=${KEY_qSTORE_PATH}<br>app.kafka.consumerTrustStorePath=${KEY_qSTORE_PATH}<br>app.kafka.consumerSslPassword=${SSL_PASSWORD}<br>app.kafka.consumerTrustStorePassword=${TRUST_STORE_PASSWORD}<br><br><br>app.kafka.producerKeyStorePath=${KEY_qSTORE_PATH}<br>app.kafka.producerTrustStorePath=${TRUST_STORE_PATH}<br>app.kafka.producerSslPassword=${SSL_PASSWORD}<br>app.kafka.producerTrustStorePassword=${TRUST_STORE_PASSWORD}<br><br>#app.kafka.bootstrapServers=${BOOT_STRAP_SERVERS:lkc-m2zxwx.dom4g2xoz0p.us-east-1.aws.confluent.cloud:9092}<br>app.kafka.bootstrapServers=${BOOT_STRAP_SERVERS:lkc-9p10qv.domqwxjl73g.us-east-1.aws.confluent.cloud:9092}<br>#app.kafka.schemaRegistryUrlConfig=${SCHEMA_REGISTRY_URL:https://psrc-8qvw0.us-east-1.aws.confluent.cloud}<br>app.kafka.consumer.group-id=${CONSUMER_GROUP_ID:revenue-management.yield-management.groupblockcollation.consumer-group}<br>app.kafka.autoRegisterSchemas=${AUTO_REGISTER_SCHEMA:false}<br><br><br>#Below are specific for kafka jobs to use start-kit<br>app.kafka.source.topic=commerce.reservation.group-block.sync<br>app.kafka.destination.topic=revenue-management.yield-management.group-block.merge<br>app.kafka.recoverableErrorTopic=commerce.reservation.group-block.collation.recoverable.groupblockcollation.dlt<br>app.kafka.irrecoverableErrorTopic=commerce.reservation.group-block.collation.irrecoverable.groupblockcollation.dlt<br>app.kafka.startingOffsets=latest<br>app.kafka.pollTimeoutMs=1000<br>app.kafka.maxOffsetPerTrigger=1000<br><br>app.kafka.sasl.oauthbearer.token.endpoint.url=${SASL_ENDPOINT:https://gatewaydsaptst1.marriott.com/v3/auth/cc-token}<br>app.kafka.schema.registry.url=${SCHEMA_REGISTRY_URL:https://psrc-8qvw0.us-east-1.aws.confluent.cloud}<br><br>app.kafka.sasl.jaas.config.clientSecret.producer=${SASL_CLIENT_SECRET_PRODUCER}<br>app.kafka.sasl.jaas.config.clientId.producer=${SASL_CLIENT_ID_PRODUCER:eeh_revmgmt_yieldmgmt}<br>app.kafka.sasl.jaas.config.extension_identityPoolId.producer=${SASL_POOL_ID_PRODUCER:pool-R7N4}<br>app.kafka.sasl.jaas.config.extension_logicalCluster.producer=${SASL_LOGICAL_CLUSTER_PRODUCER:lkc-9p10qv}<br><br>app.kafka.sasl.jaas.config.clientSecret.consumer=${SASL_CLIENT_SECRET_CONSUMER}<br>app.kafka.sasl.jaas.config.clientId.consumer=${SASL_CLIENT_ID_CONSUMER:eeh_revmgmt_yieldmgmt}<br>app.kafka.sasl.jaas.config.extension_identityPoolId.consumer=${SASL_POOL_ID_CONSUMER:pool-R7N4}<br>app.kafka.sasl.jaas.config.extension_logicalCluster.consumer=${SASL_LOGICAL_CLUSTER_CONSUMER:lkc-9p10qv}<br>#app.kafka.sasl.jaas.scope=${SASL_JAAS_SCOPE:produce consume}<br>#APP_KAFKA_SASL_JAAS_SCOPE=produce consume<br><br>#app.kafka.schema.registry.logicalCluster.avro.serializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_AVRO_SERIALIZER:lsrc-oxzwwx}<br>app.kafka.schema.registry.logicalCluster.avro.serializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_AVRO_SERIALIZER:lsrc-ok017y}<br>#app.kafka.schema.registry.logicalCluster.avro.deserializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_AVRO_DESERIALIZER:lsrc-oxzwwx}<br>app.kafka.schema.registry.logicalCluster.avro.deserializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_AVRO_DESERIALIZER:lsrc-ok017y}<br>app.kafka.schema.registry.logicalCluster.json.serializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_JSON_SERIALIZER:lsrc-oxzwwx}<br>app.kafka.schema.registry.logicalCluster.json.deserializer=${SCHEMA_REGISTRY_LOGICAL_CLUSTER_JSON_DESERIALIZER:lsrc-oxzwwx}<br><br>app.kafka.sasl.jaas.config.scope.producer=${SASL_SCOPE_PRODUCER:produce consume}<br>app.kafka.sasl.jaas.config.scope.consumer=${SASL_SCOPE_CONSUMER:produce consume}<br><br>app.kafka.schema.registry.url.serializer=https://psrc-8qvw0.us-east-1.aws.confluent.cloud<br>app.kafka.schema.registry.url.deserializer=https://psrc-8qvw0.us-east-1.aws.confluent.cloud<br>dao<br>datasource<br>entity<br>exceptions<br>helper_previous<br><br>helper_new<br><br><br>jobs<br>udfs<br><br><br><br><br>dao<br><br>datasource<br><br>groupblock.scala<br><br>package com.marriott.mgpapp.spark.groupblock.collation.dao<br><br>import java.sql.{Date, Timestamp}<br><br>import slick.driver.PostgresDriver.api._<br>import slick.lifted.ProvenShape<br><br>/**<br> * This case class used to store the GroupBlockRow Data to persist into PostGreSQL<br> */<br><br>case class GroupBlockRow(pgb_prop_code: String,<br>                        pgb_group_code: String,<br>                         pgb_cnfrm_num: Option[Int],<br>                        pgb_grp_ctrct_ind: String,<br>                        pgb_start_dt: Option[Date],<br>                        pgb_end_dt: Option[Date],<br>                        pgb_oppty_id: String,<br>                        pgb_quote_id: String,<br>                        pgb_group_name: String,<br>                        pgb_transmission_ts: Option[Timestamp],<br>                        pgb_verb: String,<br>                        pgb_last_up_by: String,<br>                        pgb_last_up_ts: Timestamp)<br><br>/**<br> * This class will be used the Map the GroupBlockRow variables with pj_gb postgreSQL DB Table.<br> *<br> * @param _tableTag<br> *<br> */<br>class GroupBlock(_tableTag: Tag, schemaName: Option[String], tableName: String)<br>  extends Table[GroupBlockRow](_tableTag, schemaName, tableName) {<br><br>  override def * : ProvenShape[GroupBlockRow] = (<br>    pgb_prop_code,<br>    pgb_group_code,<br>    pgb_cnfrm_num,<br>    pgb_grp_ctrct_ind,<br>    pgb_start_dt,<br>    pgb_end_dt,<br>    pgb_oppty_id,<br>    pgb_quote_id,<br>    pgb_group_name,<br>    pgb_transmission_ts,<br>    pgb_verb,<br>    pgb_last_up_by,<br>    pgb_last_up_ts) &lt;&gt; (GroupBlockRow.tupled, GroupBlockRow.unapply)<br><br>  def pgb_prop_code = column[String](&quot;pgb_prop_code&quot;, O.PrimaryKey, O.Length(5, varying = true))<br><br>  def pgb_group_code = column[String](&quot;pgb_group_code&quot;, O.PrimaryKey, O.Length(3, varying = true))<br><br>  def pgb_cnfrm_num = column[Option[Int]](&quot;pgb_cnfrm_num&quot;, O.PrimaryKey)<br><br>  def pgb_grp_ctrct_ind = column[String](&quot;pgb_grp_ctrct_ind&quot;,O.PrimaryKey, O.Length(1, varying = true))<br><br>  def pgb_start_dt = column[Option[Date]](&quot;pgb_start_dt&quot;)<br><br>  def pgb_end_dt = column[Option[Date]](&quot;pgb_end_dt&quot;)<br><br>  def pgb_oppty_id = column[String](&quot;pgb_oppty_id&quot;, O.Length(20, varying = true))<br><br>  def pgb_quote_id = column[String](&quot;pgb_quote_id&quot;, O.Length(20, varying = true))<br><br>  def pgb_group_name = column[String](&quot;pgb_group_name&quot;, O.Length(20, varying = true))<br><br>  def pgb_transmission_ts = column[Option[Timestamp]](&quot;pgb_transmission_ts&quot;)<br><br>  def pgb_verb = column[String](&quot;pgb_verb&quot;, O.Length(1, varying = true))<br><br>  def pgb_last_up_by = column[String](&quot;pgb_last_up_by&quot;, O.Length(50, varying = true))<br><br>  def pgb_last_up_ts = column[Timestamp](&quot;pgb_last_up_ts&quot;)<br><br>}<br><br><br><br><br>package com.marriott.mgpapp.spark.groupblock.collation.datasource<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{RecoverableErrorTopic, defaultBucket}<br>import com.marriott.elevate.spark.framework.datasource.PostgresSqlDataSourceWO<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.groupblock.Constants._<br>import com.marriott.mgpapp.spark.groupblock.collation.dao.{GroupBlock, GroupBlockRow}<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.CustomloggerMetadata<br>import com.marriott.mgpapp.spark.groupblock.collation.exceptions.PostgresException<br>import com.marriott.rm.platform.util.StreamProducerHelper<br>import com.marriott.utility.CustomLogger<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark<br>import org.apache.spark.sql.{DataFrame, ForeachWriter, SparkSession}<br>import org.postgresql.util.PSQLException<br>import slick.driver.PostgresDriver.api._<br>import slick.lifted.TableQuery<br><br>import javax.sql.DataSource<br>import scala.concurrent.Await<br>import scala.concurrent.ExecutionContext.Implicits.global<br>import scala.concurrent.duration.Duration<br>import scala.util.{Failure, Success, Try}<br><br>class DbSink(dbDetails: DbDetails,<br>             schemaName: String,<br>             tableName: String,<br>             jobIdentifier:String,<br>             jobName:String)(implicit sparkSession: SparkSession,<br>                             connectionConfigDetails: ConnectionConfigDetails,<br>                             jobConfigurationDetails: JobConfigurationDetails)<br>  extends ForeachWriter[(GroupBlockRow,String)] with LazyLogging with Serializable {<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br>  val customerMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, map)<br><br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  @transient private var session: Session = _<br>  @transient private var dataSource: DataSource = _<br>  @transient private var GroupBlockHandler: TableQuery[GroupBlock] = _<br>  @transient lazy val database = {<br>    PostgresSqlDataSourceWO.setConfig(dbDetails)<br>    val dataSourceEither = PostgresSqlDataSourceWO.getDataSource()<br><br>    dataSourceEither match {<br>      case Right(ds) =&gt;<br>        dataSource = ds<br>        log.info(&quot;Connected to DB!&quot;, customerMeta)<br>        Database.forDataSource(ds, Option(9999))<br><br>      case Left(error) =&gt;<br>        log.error(s&quot;Failed to obtain data source: $error&quot;, customerMeta)<br>        throw new RuntimeException(s&quot;Failed to obtain data source: $error&quot;)<br>    }<br>  }<br><br>  override def open(partitionId: Long, version: Long): Boolean = {<br>    openSession()<br>    true<br>  }<br><br>   def openSession() = {<br>    log.info(&quot;Opening DB Session - openSession()&quot;,<br>      customerMeta)<br>    try {<br>      session = database.createSession()<br>      GroupBlockHandler = TableQuery[GroupBlock]((tag: Tag) =&gt; new GroupBlock(tag, Some(schemaName), tableName))<br>    }<br>    catch {<br>      case exception: PSQLException =&gt;<br>        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + exception.printStackTrace(),<br>          customerMeta)<br>        new PostgresException(exception, customerMeta).handle()<br>    }<br>  }<br><br>  def process(row: (GroupBlockRow, String)): Unit = {<br>    val groupBlockRow = row._1<br><br>    log.info(&quot;groupBlockRow: &quot; + GroupBlockRow.unapply(groupBlockRow), customerMeta)<br><br>    val action = groupBlockRow.pgb_verb match {<br>      case Delete =&gt; GroupBlockHandler.filter(row =&gt;<br>          row.pgb_prop_code === groupBlockRow.pgb_prop_code &amp;&amp;<br>            row.pgb_group_code === groupBlockRow.pgb_group_code &amp;&amp;<br>            row.pgb_cnfrm_num === groupBlockRow.pgb_cnfrm_num)<br>        .map(row =&gt; (row.pgb_transmission_ts, row.pgb_verb, row.pgb_last_up_by, row.pgb_last_up_ts))<br>        .update((groupBlockRow.pgb_transmission_ts, groupBlockRow.pgb_verb, groupBlockRow.pgb_last_up_by, groupBlockRow.pgb_last_up_ts))<br>      case &quot;DS&quot; =&gt; DBIO.successful(())<br>      case _ =&gt; GroupBlockHandler.insertOrUpdate(groupBlockRow)<br>    }<br><br>    val future = session.database.run(action)<br><br>    future.onComplete {<br>      case Success(_) =&gt; log.info(&quot;Record upserted in DB successfully!&quot;, customerMeta)<br>      case Failure(error) =&gt;<br>        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + error.printStackTrace(), customerMeta)<br>        new PostgresException(error, customerMeta).handle()<br>    }<br>  }<br><br><br>  override def close(errorOrNull: Throwable): Unit = {<br>      closeSession()<br>    }<br><br>    private def closeSession(): Unit = {<br>      log.info(&quot;Closing DB Session : closeSession()&quot;,<br>        customerMeta)<br>      session.close()<br>    }<br><br>  def writeErrorToKafka(errorDS: DataFrame, topic: String, checkpoint: String): Unit = {<br>    log.info(&quot;Writing error to kafka : writeErrorToKafka()&quot;, customerMeta)<br>    val checkPointDirectory = Some(checkpoint)<br>    StreamProducerHelper.writeStreamWithParams(errorDS, checkPointPath = checkPointDirectory,<br>      topic = topic, isSingleStreaming = false)<br><br>  }<br><br>}<br><br>entity<br><br><br><br>package com.marriott.mgpapp.spark.groupblock.collation.entity<br><br>import java.sql.{Date, Timestamp}<br>import com.marriott.elevate.spark.framework.entity.{BaseEvent, DefaultEventInfo, PayLoad, StateInput}<br><br>import scala.reflect.runtime.{universe =&gt; ru}<br>import com.marriott.elevate.spark.framework.cdc.CUDPayload<br>import com.marriott.mgpapp.spark.groupblock.Constants._<br>import com.marriott.model.CustomMetadata<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{EmptyString, ErrorTrue, IrrecoverableError}<br><br>import java.time.LocalDateTime<br>import java.time.format.DateTimeFormatter<br>import scala.annotation.switch<br><br><br><br>//case class ExternalIds(opportunityID: String,<br>// quoteID: String)<br><br>case class GroupExternalItemId(id: String,<br>                                `type`: String)<br><br><br><br>case class GroupBlockEntity(actionIndicator: Option[String],<br>                            transmissionTimestamp: Option[String],<br>                            opportunityId: Option[String],<br>                            quoteId: Option[String],<br>                            confirmationNumber: Option[String],<br>                            propertyCode: Option[String],<br>                            groupCode: Option[String],<br>                            groupEffectiveDate: Option[Date],<br>                            groupExpireDate: Option[Date],<br>                            groupEventName: Option[String],<br>                            groupEventType: Option[String],<br>                            inventoryTypeCode: Option[String],<br>                            ratePlanCode: Option[String],<br>                            ratePlanEffectiveDate: Option[Date],<br>                            ratePlanExpireDate: Option[Date],<br>                            amount: Option[Int],<br>                            currencyCode: Option[String],<br>                            decimalPrecision: Option[Int],<br>                            blockedRooms: Option[Int],<br>                            inventoryEffectiveDate: Option[Date],<br>                            inventoryExpireDate: Option[Date])<br><br><br><br><br>case class GroupBlockEntityPropCode(actionIndicator: Option[String],<br>                                    transmissionTimestamp: Option[String],<br>                                    opportunityId: Option[String],<br>                                    quoteId: Option[String],<br>                                    confirmationNumber: Option[String],<br>                                    propertyCode: Option[String],<br>                                    groupCode: Option[String],<br>                                    groupEffectiveDate: Option[Date],<br>                                    groupExpireDate: Option[Date],<br>                                    groupEventName: Option[String],<br>                                    groupEventType: Option[String],<br>                                    inventoryTypeCode: Option[String],<br>                                    ratePlanCode: Option[String],<br>                                    ratePlanEffectiveDate: Option[Date],<br>                                    ratePlanExpireDate: Option[Date],<br>                                    amount: Option[Int],<br>                                    currencyCode: Option[String],<br>                                    decimalPrecision: Option[Int],<br>                                    blockedRooms: Option[Int],<br>                                    inventoryEffectiveDate: Option[Date],<br>                                    inventoryExpireDate: Option[Date],<br>                                    ppa_prop_code:String)<br><br>case class GroupBlockEntityAdj(actionIndicator: String,<br>                               transmissionTimestamp:String,<br>                               opportunityId: String,<br>                               quoteId: String,<br>                               confirmationNumber: String,<br>                               propertyCode: String,<br>                               groupCode: String,<br>                               groupEffectiveDate: Date,<br>                               groupExpireDate: Date,<br>                               groupEventName: String,<br>                               groupEventType: String,<br>                               inventoryTypeCode: String,<br>                               ratePlanCode: String,<br>                               ratePlanEffectiveDate: Date,<br>                               ratePlanExpireDate: Date,<br>                               amount: Option[Int],<br>                               currencyCode: String,<br>                               decimalPrecision: Option[Int],<br>                               blockedRooms: Option[Int],<br>                               inventoryEffectiveDate: Date,<br>                               inventoryExpireDate: Date,<br>                               ppa_prop_code:String,<br>                               data_source_ind:String,<br>                               groupIndicator:String,<br>                               groupSourceIndicator:String,<br>                               InvDateSpan:Int,<br>                               inventoryDate:String,<br>                               Amt:Double,<br>                               AdditionalIndicator:String)<br><br>case class GroupBlockEntityNew(actionIndicator: String,<br>                               transmissionTimestamp:String,<br>                               opportunityId: String,<br>                               quoteId: String,<br>                               confirmationNumber: String,<br>                               propertyCode: String,<br>                               groupCode: String,<br>                               groupEffectiveDate: Date,<br>                               groupExpireDate: Date,<br>                               groupEventName: String,<br>                               groupEventType: String,<br>                               inventoryTypeCode: String,<br>                               ratePlanCode: String,<br>                               ratePlanEffectiveDate: Date,<br>                               ratePlanExpireDate: Date,<br>                               amount: Option[Int],<br>                               currencyCode: String,<br>                               decimalPrecision: Option[Int],<br>                               blockedRooms: Option[Int],<br>                               inventoryEffectiveDate: Date,<br>                               inventoryExpireDate: Date,<br>                               ppa_prop_code:String,<br>                               data_source_ind:String,<br>                               groupIndicator:String,<br>                               groupSourceIndicator:String,<br>                               InvDateSpan:Int,<br>                               inventoryDate:Date,<br>                               Amt:Double<br>                              )<br><br>case class GroupBlockEntityNewUpdated(actionIndicator: String,<br>                                      transmissionTimestamp:String,<br>                                      opportunityId: String,<br>                                      quoteId: String,<br>                                      confirmationNumber: String,<br>                                      propertyCode: String,<br>                                      groupCode: String,<br>                                      groupEffectiveDate: Date,<br>                                      groupExpireDate: Date,<br>                                      groupEventName: String,<br>                                      groupEventType: String,<br>                                      inventoryTypeCode: String,<br>                                      ratePlanCode: String,<br>                                      ratePlanEffectiveDate: Date,<br>                                      ratePlanExpireDate: Date,<br>                                      amount: Option[Int],<br>                                      currencyCode: String,<br>                                      decimalPrecision: Option[Int],<br>                                      blockedRooms: Option[Int],<br>                                      inventoryEffectiveDate: Date,<br>                                      inventoryExpireDate: Date,<br>                                      ppa_prop_code:String,<br>                                      data_source_ind:String,<br>                                      groupIndicator:String,<br>                                      groupSourceIndicator:String,<br>                                      inventoryDate:Date,<br>                                      Amt:Double<br>                                     )<br><br>case class GroupBlockEntityUpdatedCurrencyCode(actionIndicator: String,<br>                                               transmissionTimestamp:String,<br>                                               opportunityId: String,<br>                                               quoteId: String,<br>                                               confirmationNumber: String,<br>                                               propertyCode: String,<br>                                               groupCode: String,<br>                                               groupEffectiveDate: Date,<br>                                               groupExpireDate: Date,<br>                                               groupEventName: String,<br>                                               groupEventType: String,<br>                                               inventoryTypeCode: String,<br>                                               ratePlanCode: String,<br>                                               ratePlanEffectiveDate: Date,<br>                                               ratePlanExpireDate: Date,<br>                                               amount: Option[Int],<br>                                               currencyCode: String,<br>                                               decimalPrecision: Option[Int],<br>                                               blockedRooms: Option[Int],<br>                                               inventoryEffectiveDate: Date,<br>                                               inventoryExpireDate: Date,<br>                                               ppa_prop_code:String,<br>                                               data_source_ind:String,<br>                                               groupIndicator:String,<br>                                               groupSourceIndicator:String,<br>                                               inventoryDate:Date,<br>                                               Amt:Double<br>                                              )<br><br><br>case class PropParamAssignEntity(ppa_prop_code:Option[String],<br>                                 ppa_param_id:Option[String],<br>                                 ppa_param_value:Option[String],<br>                                 ppa_active_ind:Option[String],<br>                                 ppa_eff_start_dt:Option[Date],<br>                                 ppa_eff_end_dt:Option[Date],<br>                                 ppa_last_upd_by:Option[String],<br>                                 ppa_last_upd_ts:Option[Timestamp],<br>                                 ppa_feature_id:Option[String])<br><br><br>case class ppaPropCodeParamValue(ppa_prop_code:String,<br>                                 ppa_param_value:String)<br><br><br>case class ppaPropCodeValue(ppa_prop_code:String)<br><br>case class groupEntity( actionIndicator: String,<br>                        transmissionTimestamp:String,<br>                        opportunityId: String,<br>                        quoteId: String,<br>                        confirmationNumber: String,<br>                        propertyCode: String,<br>                        groupCode: String,<br>                        groupEffectiveDate: Date,<br>                        groupExpireDate: Date,<br>                        groupEventName: String,<br>                        groupEventType: String,<br>                        inventoryTypeCode: String,<br>                        ratePlanCode: String,<br>                        ratePlanEffectiveDate: Date,<br>                        ratePlanExpireDate: Date,<br>                        amount: Option[Int],<br>                        currencyCode: String,<br>                        decimalPrecision: Option[Int],<br>                        blockedRooms: Option[Int],<br>                        inventoryEffectiveDate: Date,<br>                        inventoryExpireDate: Date,<br>                        ppa_prop_code:String,<br>                        data_source_ind:String,<br>                        groupIndicator:String,<br>                        groupSourceIndicator:String,<br>                        InvDateSpan:String,<br>                        InventoryDateArray:String,<br>                        inventoryDate:String,<br>                        Amt:Double<br>                        //ActionIndicatornw:String<br>                      )<br><br>case class pj_gb(pgb_prop_code: Option[String],<br>                 pgb_group_code: Option[String],<br>                 pgb_Seq_num: Option[String],<br>                 quoteId: Option[String],<br>                 pgb_transmission_ts: Option[Timestamp]<br>                )<br>final case class CustomloggerMetadata(<br>                                       traceId: String,<br>                                       spanId: String,<br>                                       project: String,<br>                                       app: String,<br>                                       flow: String,<br>                                       customData: Map[String,String]) extends CustomMetadata with Serializable<br><br>case class GroupBlockCollationEvent(groupBlockInfo: DefaultEventInfo,<br>                                    groupBlockPayload: GBCollationPayload)<br>  extends BaseEvent[DefaultEventInfo, GBCollationPayload] with Serializable{<br><br>  override def getEventInfo: DefaultEventInfo = groupBlockInfo<br><br>  override def getPayload: GBCollationPayload = groupBlockPayload<br><br>  override def eType: ru.TypeTag[DefaultEventInfo] = ru.typeTag[DefaultEventInfo]<br><br>  override def pType: ru.TypeTag[GBCollationPayload] = ru.typeTag[GBCollationPayload]<br><br>}<br><br>case class GBCollationPayload(payLoad: String = null,<br>                              payLoadType: String = &quot;JSON&quot;,<br>                              groupBlockCollation:Option[GroupBlockCollation],<br>                              groupBlockCUD:CUDPayload) extends PayLoad with Serializable<br><br><br>case class GroupBlockCollationSchema(<br>                                      groupCode: String,<br>                                      propertyCode: String,<br>                                      confirmationNumber: String,<br>                                      groupEffectiveDate: Date,<br>                                      groupExpireDate: Date,<br>                                      groupInd: String,<br>                                      groupIndicator: String,<br>                                      opportunityId: String,<br>                                      quoteId: String,<br>                                      groupEventName: String,<br>                                      transmissionTimestamp: String,<br>                                      currencyCode: String,<br>                                      actionIndicator: String,<br>                                      inventoryDate: Date,<br>                                      inventoryTypeCode: String,<br>                                      blockedRooms: Int = 0,<br>                                      avgAmt: Double,<br>                                      numRecs: Long,<br>                                      recordTimestamp: String,<br>                                      blockedRevenue: Double = 0.0<br>                                    )<br><br><br><br><br>case class GroupBlockCollation(actionEnum:String,<br>                               generationTime:String,<br>                               dateTimeStamp: Option[DateTimeStamp],<br>                             //  groupIds:GroupId,<br>                               hotel:Hotel,<br>                               products:Seq[InventoryTypeDetails],<br>                               groupContract:GroupContract,<br>                               offer:Offer,<br>                               pgb_transmission_ts:Option[String],<br>                               pgb_start_dt:Option[String],<br>                               pgb_end_dt:Option[String]) extends Serializable<br><br>case class DateTimeStamp(create:Option[String],<br>                         creationDateTimeHotelLocalTime:Option[String],<br>                         lastModify:Option[String],<br>                         lastUpdateDateTimeHotelLocalTime:Option[String],<br>                         purge:Option[String],<br>                         cancel:Option[String],<br>                         cancellationDateTimeHotelLocalTime:Option[String])<br><br>//case class GroupId(opportunityID:String,<br>                  // quoteID:String)<br><br>case class Hotel(propertyCode:PropertyCode)<br><br>case class PropertyCode(code:String<br>  )<br><br>case class InventoryTypeDetails(inventoryTypeCode:String,<br>                                baseOccupancy: Int,<br>                                inventoryRefId: Int,<br>                                ratePlanRefIds: Option[Int])<br><br><br><br>case class GroupContract(code:String,<br>                         name:Seq[Description],<br>                         timespan:Timespan,<br>                         confirmationNumber:ProviderStateId,<br>                         groupType:String,<br>                        //inventory:GroupInventory,<br>                         pricePlans:Seq[PricePlanDetails],<br>                         rateReferences:Option[Seq[RateReference]],<br>                         externalIds: Seq[GroupExternalItemId]<br>                         )<br><br>case class RateReference(ratePlanCode: Option[String])<br>case class Description(text:Option[Seq[String]])<br>case class Timespan(start:String,<br>                    end:String,<br>                    duration:Option[String])<br><br>case class ProviderStateId(value:String)<br><br><br><br>//case class GroupInventory(countersPerDays:Option[Seq[CountersPerDay]])<br><br>//case class CountersPerDay(inventoriesRefID:Int,<br>                 //          date:String,<br>                 //         deducted:Int)<br>//case class PricePlan(priceGridDetails:PriceGridDetails)<br>case class Offer(productUses:Option[Seq[ProductUses]])<br><br><br>  case class ProductUses(<br>                          timespan: Timespan,<br>                          inventoryTypeCode: String,<br>                          countersPerInventoryType: CountersPerInventoryType<br>                        )<br><br>  case class CountersPerInventoryType(<br>                                       deducted: Int<br>                                     )<br><br><br><br><br><br>case class PricePlanDetails(priceGridDetails:PriceGridDetails)<br><br>case class PriceGridDetails(amountRules:Seq[AmountRules])<br><br>case class AmountRules(inventoryTypes: Seq[InventoryType],<br>                       timespan: Timespan)<br><br>case class InventoryType(inventoryTypeCode:String,<br>                          amountValue:Option[CurrencyAmount],<br>                         guestsByType:Option[GuestByType])<br><br>case class CurrencyAmount(currencyCode:String,<br>                          value:Int,<br>                          numberOfDecimals:Int)<br>case class GuestByType(guestType:Option[String],<br>                        count:Option[Int])<br><br><br>case class GroupBlockData(pgb_prop_code: String,<br>                          pgb_group_code: String,<br>                          pgb_cnfrm_num: Int,<br>                          pgb_grp_ctrct_ind: String,<br>                          pgb_start_dt: Option[Date],<br>                          pgb_end_dt: Option[Date],<br>                          pgb_oppty_id: String,<br>                          pgb_quote_id: String,<br>                          pgb_group_name: String,<br>                          pgb_transmission_ts: Option[Timestamp],<br>                          pgb_verb: String,<br>                          pgb_last_up_by: String)<br><br>//case class EEHData(<br>//                    timeStamp: String,<br>//                    metadata: Map[String, String],<br>//                    retryCounter: Int,<br>//                    payLoad: String,<br>//                    eventVersion: String,<br>//                    eventSource: String,<br>//                    destinationTopic: String,<br>//                    eventIdentifier: String,<br>//                    payLoadType: String,<br>//                    recordNum: String,<br>//                    error: String,<br>//                    errorType: Seq[String],<br>//                    message: Seq[String],<br>//                    warning: Seq[String]<br>//                  )<br><br>case class GroupBlockKafkaEvent(timeStamp: String,<br>                                retryCounter: Int = 0,<br>                                payLoad: String,<br>                                eventSource: String,<br>                                destinationTopic: String,<br>                                eventIdentifier: String,<br>                                payLoadType: String,<br>                                recordNum: String,<br>                                error: String,<br>                                errorType: Seq[String],<br>                                message: Seq[String],<br>                                warning: Seq[String]) extends StateInput<br><br><br>final case class ReservationGroupBlockMerge(var timeStamp: Option[String], var metadata: Map[String, Option[String]], var retryCounter: Option[Int], var payLoad: Option[String], var eventVersion: Option[String], var eventSource: Option[String], var destinationTopic: Option[String], var eventIdentifier: Option[String], var payLoadType: Option[String], var recordNum: Option[String], var error: Option[String], var errorType: Seq[Option[String]], var message: Seq[Option[String]], var warning: Seq[Option[String]]) extends org.apache.avro.specific.SpecificRecordBase {<br>  def this() = this(None, Map(&quot;&quot; -&gt; None), None, None, None, None, None, None, None, None, None, Seq.empty, Seq.empty, Seq.empty)<br>  def get(field$: Int): AnyRef = {<br>    (field$: @switch) match {<br>      case 0 =&gt; {<br>        timeStamp match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 1 =&gt; {<br>        val map: java.util.HashMap[String, Any] = new java.util.HashMap[String, Any]<br>        metadata foreach { kvp =&gt;<br>          val key = kvp._1<br>          val value = kvp._2<br>          map.put(key, value match {<br>            case Some(x) =&gt; x<br>            case None =&gt; null<br>          })<br>        }<br>        map<br>      }.asInstanceOf[AnyRef]<br>      case 2 =&gt; {<br>        retryCounter match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 3 =&gt; {<br>        payLoad match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 4 =&gt; {<br>        eventVersion match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 5 =&gt; {<br>        eventSource match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 6 =&gt; {<br>        destinationTopic match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 7 =&gt; {<br>        eventIdentifier match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 8 =&gt; {<br>        payLoadType match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 9 =&gt; {<br>        recordNum match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 10 =&gt; {<br>        error match {<br>          case Some(x) =&gt; x<br>          case None =&gt; null<br>        }<br>      }.asInstanceOf[AnyRef]<br>      case 11 =&gt; {<br>        scala.collection.JavaConverters.bufferAsJavaListConverter({<br>          errorType map {<br>            case Some(x) =&gt; x<br>            case None =&gt; null<br>          }<br>        }.toBuffer).asJava<br>      }.asInstanceOf[AnyRef]<br>      case 12 =&gt; {<br>        scala.collection.JavaConverters.bufferAsJavaListConverter({<br>          message map {<br>            case Some(x) =&gt; x<br>            case None =&gt; null<br>          }<br>        }.toBuffer).asJava<br>      }.asInstanceOf[AnyRef]<br>      case 13 =&gt; {<br>        scala.collection.JavaConverters.bufferAsJavaListConverter({<br>          warning map {<br>            case Some(x) =&gt; x<br>            case None =&gt; null<br>          }<br>        }.toBuffer).asJava<br>      }.asInstanceOf[AnyRef]<br>      case _ =&gt; new org.apache.avro.AvroRuntimeException(&quot;Bad index&quot;)<br>    }<br>  }<br>  def put(field$: Int, value: Any): Unit = {<br>    (field$: @switch) match {<br>      case 0 =&gt; this.timeStamp = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 1 =&gt; this.metadata = {<br>        value match {<br>          case (map: java.util.Map[_,_]) =&gt; {<br>            scala.collection.JavaConverters.mapAsScalaMapConverter(map).asScala.toMap map { kvp =&gt;<br>              val key = kvp._1.toString<br>              val value = kvp._2<br>              (key, value match {<br>                case null =&gt; None<br>                case _ =&gt; Some(value.toString)<br>              })<br>            }<br>          }<br>        }<br>      }.asInstanceOf[Map[String, Option[String]]]<br>      case 2 =&gt; this.retryCounter = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value)<br>        }<br>      }.asInstanceOf[Option[Int]]<br>      case 3 =&gt; this.payLoad = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 4 =&gt; this.eventVersion = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 5 =&gt; this.eventSource = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 6 =&gt; this.destinationTopic = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 7 =&gt; this.eventIdentifier = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 8 =&gt; this.payLoadType = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 9 =&gt; this.recordNum = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 10 =&gt; this.error = {<br>        value match {<br>          case null =&gt; None<br>          case _ =&gt; Some(value.toString)<br>        }<br>      }.asInstanceOf[Option[String]]<br>      case 11 =&gt; this.errorType = {<br>        value match {<br>          case (array: java.util.List[_]) =&gt; {<br>            scala.collection.JavaConverters.asScalaIteratorConverter(array.iterator).asScala.map({<br>              case null =&gt; None<br>              case x =&gt; Some(x.toString)<br>            }).toSeq<br>          }<br>        }<br>      }.asInstanceOf[Seq[Option[String]]]<br>      case 12 =&gt; this.message = {<br>        value match {<br>          case (array: java.util.List[_]) =&gt; {<br>            scala.collection.JavaConverters.asScalaIteratorConverter(array.iterator).asScala.map({<br>              case null =&gt; None<br>              case x =&gt; Some(x.toString)<br>            }).toSeq<br>          }<br>        }<br>      }.asInstanceOf[Seq[Option[String]]]<br>      case 13 =&gt; this.warning = {<br>        value match {<br>          case (array: java.util.List[_]) =&gt; {<br>            scala.collection.JavaConverters.asScalaIteratorConverter(array.iterator).asScala.map({<br>              case null =&gt; None<br>              case x =&gt; Some(x.toString)<br>            }).toSeq<br>          }<br>        }<br>      }.asInstanceOf[Seq[Option[String]]]<br>      case _ =&gt; new org.apache.avro.AvroRuntimeException(&quot;Bad index&quot;)<br>    }<br>    ()<br>  }<br>  def getSchema: org.apache.avro.Schema = ReservationGroupBlockMerge.SCHEMA$<br>}<br>object ReservationGroupBlockMerge {<br>  val SCHEMA$ = new org.apache.avro.Schema.Parser().parse(&quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;ReservationGroupBlockMerge\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;timeStamp\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;metadata\&quot;,\&quot;type\&quot;:{\&quot;type\&quot;:\&quot;map\&quot;,\&quot;values\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]}},{\&quot;name\&quot;:\&quot;retryCounter\&quot;,\&quot;type\&quot;:[\&quot;int\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;payLoad\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;eventVersion\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;eventSource\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;destinationTopic\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;eventIdentifier\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;payLoadType\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;recordNum\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;error\&quot;,\&quot;type\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]},{\&quot;name\&quot;:\&quot;errorType\&quot;,\&quot;type\&quot;:{\&quot;type\&quot;:\&quot;array\&quot;,\&quot;items\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]}},{\&quot;name\&quot;:\&quot;message\&quot;,\&quot;type\&quot;:{\&quot;type\&quot;:\&quot;array\&quot;,\&quot;items\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]}},{\&quot;name\&quot;:\&quot;warning\&quot;,\&quot;type\&quot;:{\&quot;type\&quot;:\&quot;array\&quot;,\&quot;items\&quot;:[\&quot;string\&quot;,\&quot;null\&quot;]}}]}&quot;)<br><br>}<br><br>exceptions<br><br>package com.marriott.mgpapp.spark.groupblock.collation.exceptions<br><br>import com.marriott.mgpapp.spark.groupblock.Constants.{customErrorMessage, JobIdentifier, JobName, loggerAppName}<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.CustomloggerMetadata<br>import com.marriott.rm.platform.exception.`enum`.ErrorCode<br>import com.marriott.rm.platform.exception.event.BaseException<br>import com.marriott.utility.CustomLogger<br>import org.apache.avro.SchemaParseException<br>import org.apache.kafka.common.errors.{InvalidTopicException, OutOfOrderSequenceException, TopicAuthorizationException}<br><br>class KafkaException(errorCause: Throwable) extends BaseException {<br>  val log = new CustomLogger(this.getClass.getName)<br><br>  override def getErrorCode: String = errorCause match {<br>    case ex: TopicAuthorizationException =&gt; ErrorCode.EventTypeInvalid403.toString<br>    case ex: SchemaParseException =&gt; ErrorCode.BadRequest400.toString<br>    case ex: InvalidTopicException =&gt; ErrorCode.BadRequest400.toString<br>    case ex: OutOfOrderSequenceException =&gt; ErrorCode.BadRequest400.toString<br>    case _ =&gt; ErrorCode.InternalServerError500.toString<br>  }<br><br>  override def getErrorMessage: String = errorCause.getMessage<br><br>  override def getErrorCause: Throwable = errorCause<br><br>  override def handle(input: Map[String, Any]): Boolean = {<br>    val jobIdentifierStr: String =<br>      input.getOrElse(JobIdentifier, JobIdentifier).toString<br>    val jobNameStr: String = input.getOrElse(JobName, JobName).toString<br>    val customErrMessage = input.getOrElse(customErrorMessage, &quot;&quot;)<br>    log.error(<br>      customErrMessage + getErrorMessage,<br>      CustomloggerMetadata(<br>        &quot;traceId&quot;,<br>        &quot;spanId&quot;,<br>        loggerAppName,<br>        jobIdentifierStr,<br>        jobNameStr,<br>        Map.empty[String, String]<br>      )<br>    )<br>    true<br>  }<br><br>}<br><br><br><br><br>package com.marriott.mgpapp.spark.groupblock.collation.exceptions<br><br>import com.marriott.model.CustomMetadata<br>import com.marriott.rm.platform.exception.event.BaseException<br>import com.marriott.utility.CustomLogger<br><br>/**<br> *<br> * @param errorCode  : Error Code for the exception<br> * @param errorCause : Actual cause of the exception<br> */<br><br>class PostgresException(errorCause: Throwable, customMeta: CustomMetadata) extends BaseException {<br>  private val log: CustomLogger = new CustomLogger(this.getClass.getName)<br><br>  override def getErrorCode: String = &quot;503&quot;<br><br>  override def getErrorMessage: String = errorCause.getMessage<br><br>  override def getErrorCause: Throwable = errorCause<br><br>  override def handle(input: Map[String, Any] = Map.empty): Boolean = {<br>    log.info(<br>      s&quot;&quot;&quot;<br>         |Exception has occurred in starter kit<br>         |Error Code = $getErrorCode<br>         |Error Message = $getErrorMessage<br>         |Error Cause = ${getErrorCause.getClass.toString}&quot;&quot;&quot;.stripMargin,<br>      customMeta<br>    )<br>    if (input.isEmpty) {<br>      log.info(&quot;&quot;&quot;Error Handler Input Map is Empty&quot;&quot;&quot;.stripMargin, customMeta)<br>    } else {<br>      log.info(&quot;&quot;&quot;Error Handler Input Map Key Value Pairs given below&quot;&quot;&quot;.stripMargin, customMeta)<br>      input.foreach(obj =&gt; {<br>        log.info(s&quot;&quot;&quot;${obj._1} -&gt; ${obj._2}&quot;&quot;&quot;.stripMargin, customMeta)<br>      })<br>    }<br>    true<br>  }<br>}<br><br><br><br><br><br>helper<br><br>groupblockfiltration<br><br>package com.marriott.mgpapp.spark.groupblock.collation.helper<br><br>import com.fasterxml.jackson.databind.exc.MismatchedInputException<br>import com.fasterxml.jackson.databind.{DeserializationFeature, JsonMappingException}<br>import com.marriott.elevate.spark.framework.cdc.{CUDBaseTable, CUDPayload}<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{ErrorTrue, IrrecoverableError, IrrecoverableErrorTopic, RecoverableErrorTopic, SourceTopic, defaultBucket}<br>import com.marriott.elevate.spark.framework.entity.DefaultEventInfo<br>import com.marriott.elevate.spark.framework.traits.KafkaSSLOptions<br>import com.marriott.elevate.spark.framework.utils.{JsonUtil, S3Util}<br>import com.marriott.mgpapp.spark.groupblock.Constants.{Default, EventIdentifierValue, EventSourceValue, GroupBlockCollationIrrecoverableEntityCheckpointDir, GroupBlockCollationRecoverableEntityCheckpointDir, PayLoadTypeValue, RetryCounterValue}<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.marriott.rm.platform.constants.KafkaProperties.{irrecoverableErrorTopic, recoverableErrorTopic}<br>import com.marriott.rm.platform.util.{InputType, StreamProducerHelper}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import com.marriott.utility.CustomLogger<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br>import org.apache.spark.sql.types.StructType<br>import org.apache.commons.lang3.exception.ExceptionUtils<br>import org.apache.spark.sql.streaming.OutputMode<br>import org.apache.spark.sql.streaming.Trigger<br><br><br>import scala.io.Source<br><br><br>trait GroupBlockFiltrationHelper extends LazyLogging with Serializable {<br><br>  val Json = &quot;json&quot;<br>  @transient val log: CustomLogger<br>  val customerMeta: CustomloggerMetadata<br>  val kafkaTopicMap: Option[Map[String, String]]<br><br><br>  def groupBlockFiltration(kafkaData: DataFrame)<br>                          (implicit spark: SparkSession,<br>                           connectionConfigDetails: ConnectionConfigDetails,<br>                           jobConfigurationDetails: JobConfigurationDetails): Dataset[GroupBlockCollationEvent] = {<br>    //kafkaData.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    import spark.implicits._<br><br>   // println(&quot;file testing&quot; )<br>    //val strings = kafkaData.map(x =&gt; x.getString(0))<br>    //strings.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    val groupBlockDataset = kafkaData.map(x =&gt; convertToEntityGroupBlockCollation(x.getString(0)))<br>    println(&quot;----convert to Entity check ----&quot;)<br>     groupBlockDataset.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br><br>    val groupBlockDatasetValid = groupBlockDataset.filter(x =&gt; x._1.isDefined)<br>    val groupBlockDatasetInvalid = groupBlockDataset.filter(x =&gt; x._1.isEmpty)<br>    //    groupBlockDatasetInvalid.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br><br>    println(&quot;----Schema check check ----&quot;)<br>    groupBlockDatasetValid.printSchema()<br>    val groupBlockDF = groupBlockDatasetValid.toDF(&quot;value&quot;, &quot;payload&quot;).drop(&quot;payload&quot;)<br>    groupBlockDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    groupBlockDF.printSchema()<br>    //    val groupBlockDF = groupBlockAndIrrDf.filter(col(&quot;value&quot;).isNotNull).select(&quot;value&quot;)<br>    log.info(&quot;DataFrame to Entity Conversion&quot;, customerMeta)<br>    val column: Column = col(&quot;value.hotel&quot;)<br>      .getItem(&quot;propertyCode&quot;)<br>      .getItem(&quot;code&quot;)<br>      .alias(&quot;property-code&quot;)<br><br>    val (mgpPropertyDf, nonMgpPropertyDf) = FilterPropertyCodeUtil.filterPropertyCode(groupBlockDF, column)<br>    println(&quot;---Filter Property check ----&quot;)<br>    mgpPropertyDf.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br><br>    println(&quot;---Non mgp Filter Property check ----&quot;)<br>    nonMgpPropertyDf.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    val irrecoverableDF = groupBlockDatasetInvalid.toDF(&quot;value&quot;, &quot;payload&quot;).drop(&quot;value&quot;).withColumnRenamed(&quot;payload&quot;, &quot;value&quot;)<br>    //      .union(nonMgpPropertyDf)<br>    // irrecoverable<br>    //    irrecoverableDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br><br>    // Mapping Dataframe to columnmap and to Dataset<br><br>    val transactionformedDataDF = transformtoDF(mgpPropertyDf)<br>    //transactionformedDataDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    val groupBlockCollationDS = transactionformedDataDF.map(row =&gt; convertToFinalDS(row))<br><br>   // val DLQcheckPoint = &quot;C:\\Checkpoint\\DLQ&quot;<br>    //    val DLQcheckPoint = S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(defaultBucket), jobConfigurationDetails.jobname, jobConfigurationDetails.checkpointdirectory.get(GroupBlockCollationIrrecoverableEntityCheckpointDir))<br>    //    writeErrorToKafka(irrecoverableDF,topic = jobConfigurationDetails.kafkatopicmap.get(IrrecoverableErrorTopic),checkpoint = DLQcheckPoint)<br><br>    val DLQcheckPoint = S3Util.getCheckPointDirectory(<br>      connectionConfigDetails.bucketconfig.get(Default),<br>      jobConfigurationDetails.jobname,<br>      checkPointDirectory(GroupBlockCollationIrrecoverableEntityCheckpointDir)<br>    // returning Dataset[GroupBlockCollationEvent]<br>    groupBlockCollationDS<br>  }<br><br><br>  def convertToFinalDS(row: Row): GroupBlockCollationEvent = {<br><br>    //    log.info(&quot;convertToFinalDS :: Converting dataframe to columnmap - CUD-PAYLOAD to GroupBlockCollationEvent Dataset&quot;, customerMeta)<br><br>    val columns = Seq(&quot;actionEnum&quot;, &quot;generationTime&quot;, &quot;confirmationNumber&quot;, &quot;opportunityId&quot;, &quot;quoteId&quot;, &quot;propertyCode&quot;, &quot;groupCode&quot;, &quot;groupName&quot;, &quot;groupType&quot;, &quot;groupEffectiveDate&quot;, &quot;groupExpireDate&quot;, &quot;ratePlanEffectiveDate&quot;, &quot;ratePlanExpireDate&quot;, &quot;inventoryTypeCode&quot;, &quot;ratePlanCode&quot;, &quot;inventoryEffectiveDate&quot;, &quot;inventoryExpireDate&quot;, &quot;verb&quot;, &quot;amount&quot;, &quot;currencyCode&quot;, &quot;dec_pre&quot;, &quot;blocked&quot;, &quot;HotelId&quot;)<br>    val coloumnmap: Map[String, String] = columns.map { fieldName =&gt;<br>      val columnName = fieldName<br>      val value = Option(row.getAs[Any](columnName)).map(_.toString).getOrElse(&quot;null&quot;)<br>      columnName -&gt; value<br>    }.toMap<br><br>    val cUDBaseTable: CUDBaseTable = CUDBaseTable(&quot;&quot;, coloumnmap(&quot;HotelId&quot;), &quot;&quot;, coloumnmap(&quot;generationTime&quot;), coloumnmap(&quot;verb&quot;))<br>    val cudentity: CUDPayload = CUDPayload(cUDBaseTable, coloumnmap, &quot;&quot;, &quot;&quot;)<br>    val GBpayload: GBCollationPayload = GBCollationPayload(&quot;&quot;, &quot;&quot;, null, cudentity)<br><br>    GroupBlockCollationEvent(groupBlockInfo = null, groupBlockPayload = GBpayload)<br>  }<br><br><br>  def convertToEntityGroupBlockCollation(entity: String)<br>                                        (implicit sparkSession: SparkSession): (Option[GroupBlockCollation], String) = {<br>    log.info(&quot;Starting the process of parsing the source data as per the entities mentioned in the [GroupBlock] - convertToEntityGroupBlockCollation()&quot;, customerMeta)<br><br>    val filePath = &quot;C:\\Users\\ajune750\\AppData\\Roaming\\JetBrains\\IdeaIC2023.3\\scratches\\scratch_1.json&quot;<br>    ////        val filePath = &quot;scratch_1.json5&quot;C:\Users\lpal<br>    val entity: String = Source.fromFile(filePath).mkString<br>    try {<br>      configureJsonMapper()<br>      val data = Some(JsonUtil.fromJson[GroupBlockCollation](entity))<br>      (data, entity)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;KafkaStreamingJob ::: getEventFromString() :: Exception :&quot; + exception, customerMeta)<br>        (None, entity)<br>    }<br>  }<br><br>  private def configureJsonMapper(): Unit = {<br>    JsonUtil.mapper.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES,true)<br>    JsonUtil.mapper.configure(DeserializationFeature.FAIL_ON_NULL_CREATOR_PROPERTIES,true)<br>  }<br><br>  def writeErrorToKafka(errorDS: DataFrame, topic: String, checkpoint: String): Unit = {<br>    log.info(&quot;Writing error to kafka : writeErrorToKafka() into topic &quot; + topic + &quot; with a checkpoint &quot; + checkpoint, customerMeta)<br>    //    errorDS.writeStream.format(&quot;console&quot;).options(Map(&quot;truncate&quot; -&gt; &quot;false&quot;)).start()<br>    val checkPointDirectory = Some(checkpoint)<br>    StreamProducerHelper.writeStreamWithParams(errorDS, checkPointPath = checkPointDirectory, topic = topic, isSingleStreaming = false)<br><br>  }<br><br>  def transformtoDF(mgpData: DataFrame) = {<br>   // val schema = &quot;start string, end string&quot;<br>    val explodedDataDF = mgpData.<br>      withColumn(&quot;actionEnum&quot;, col(&quot;value.actionEnum&quot;))<br>      .withColumn(&quot;generationTime&quot;, col(&quot;value.generationTime&quot;))<br>      .withColumn(&quot;confirmationNumber&quot;, col(&quot;value.groupContract.confirmationNumber.value&quot;))<br>      //.withColumn(&quot;opportunityId&quot;, when(array_contains(col(&quot;value.groupContract.externalIds.type&quot;), &quot;OpportunityID&quot;), col(&quot;value.groupContract.externalIds.id&quot;)))<br>      //.withColumn(&quot;quoteId&quot;, when(array_contains(col(&quot;value.groupContract.externalIds.type&quot;), &quot;QuoteID&quot;), col(&quot;value.groupContract.externalIds.id&quot;)))<br>      .withColumn(&quot;externalId&quot;, explode(col(&quot;value.groupContract.externalIds&quot;)))<br>      .withColumn(&quot;opportunityId&quot;, when(col(&quot;externalId.type&quot;) === &quot;opportunityID&quot;, col(&quot;externalId.id&quot;)))<br>      .withColumn(&quot;quoteId&quot;, when(col(&quot;externalId.type&quot;) === &quot;quoteID&quot;, col(&quot;externalId.id&quot;)))<br>      //  .withColumn (&quot;opportunityID&quot;, col (&quot;value.groupContract.externalIds.type&quot;) )<br>      // .withColumn (&quot;quoteId&quot;, col (&quot;value.groupContract.externalIds.type&quot;) )<br>      //  .withColumn (&quot;opportunityID&quot;, col (&quot;value.groupIds.opportunityID&quot;) )<br>      //  .withColumn (&quot;quoteId&quot;, col (&quot;value.groupIds.quoteId&quot;) )<br>      .withColumn(&quot;propertyCode&quot;, col(&quot;value.hotel.propertyCode.code&quot;))<br>      .withColumn(&quot;groupCode&quot;, col(&quot;value.groupContract.code&quot;))<br>      .withColumn(&quot;groupName_&quot;, explode(col(&quot;value.groupContract.name.text&quot;)))<br>      .withColumn(&quot;groupName&quot;, explode(col(&quot;groupName_&quot;)))<br>      .withColumn(&quot;groupType&quot;, col(&quot;value.groupContract.groupType&quot;))<br>      .withColumn(&quot;groupEffectiveDate&quot;, col(&quot;value.groupContract.timespan.start&quot;).cast(&quot;String&quot;))<br>      .withColumn(&quot;groupExpireDate&quot;, col(&quot;value.groupContract.timespan.end&quot;).cast(&quot;String&quot;))<br>      // .withColumn (&quot;rates___&quot;, explode (col (&quot;value.groupContract.pricePlans.priceGridDetails.amountRules&quot;) ) )<br>      .withColumn(&quot;rates___&quot;, explode(col(&quot;value.groupContract.pricePlans.priceGridDetails.amountRules&quot;)))<br>      .withColumn(&quot;rates__&quot;, explode(col(&quot;rates___&quot;)))<br>      .withColumn(&quot;rates_&quot;, explode(col(&quot;rates__.inventoryTypes&quot;)))<br>      .withColumn(&quot;ratePlanEffectiveDate&quot;, explode(col(&quot;rates___.timespan.start&quot;)))<br>      .withColumn(&quot;ratePlanExpireDate&quot;, explode(col(&quot;rates___.timespan.end&quot;))) // this is related to priceplan timespan<br>     // .withColumn(&quot;counters_&quot;, explode(col(&quot;value.groupContract.inventory.countersPerDays&quot;)))<br>     .withColumn(&quot;counters_&quot;, explode(col(&quot;value.offer.productUses&quot;)))<br>      .withColumn(&quot;inventories_&quot;, explode(col(&quot;value.products&quot;)))<br>      .withColumn(&quot;inventoryTypeCode&quot;, col(&quot;inventories_.inventoryTypeCode&quot;)) //inventoryTypeCode is in products, groupcontract and offer, this one is of product<br>      .withColumn(&quot;ratePlanCode&quot;, col(&quot;value.groupContract.rateReferences.ratePlanCode&quot;))<br>      //.withColumn(&quot;inventoryEffectiveDate&quot;, col(&quot;counters_.date&quot;))<br>      //.withColumn(&quot;inventoryExpireDate&quot;, col(&quot;counters_.date&quot;))<br>      .withColumn(&quot;inventoryEffectiveDate&quot;, col(&quot;counters_.timespan.start&quot;))<br>      .withColumn(&quot;inventoryExpireDate&quot;, col(&quot;counters_.timespan.end&quot;))<br>      // Parse the counters_.timespan column and create the new columns<br>    //  .withColumn(&quot;counters_.timespan&quot;, from_json($&quot;counters_.timespan&quot;, schema))<br>    //  .withColumn(&quot;inventoryEffectiveDate&quot;, col(&quot;counters_.timespan.start&quot;))<br>    //  .withColumn(&quot;inventoryExpireDate&quot;, col(&quot;counters_.timespan.end&quot;))<br>      .drop(&quot;value&quot;, &quot;rates___&quot;, &quot;rates__&quot;, &quot;groupName_&quot;, &quot;counters_.timespan&quot;, &quot;externalId&quot;)<br>     // .drop(&quot;value&quot;, &quot;rates___&quot;, &quot;rates__&quot;, &quot;groupName_&quot;)<br>      .filter(col(&quot;rates_.guestsByType.guestType&quot;) === &quot;ADULT&quot;)<br><br><br>    log.info(<br>      s&quot;Columns of exploded dataframe :: ${explodedDataDF.columns.mkString(&quot;Array(&quot;, &quot;, &quot;, &quot;)&quot;)}&quot;,<br>      customerMeta<br>    )<br><br>    val finaldf = explodedDataDF.dropDuplicates() // TO-D0 will be cleared later<br>    // Absense Dataframe which doesn&#39;t have Rates info<br>    val absenceDataDF = finaldf.filter(col(&quot;rates_.inventoryTypeCode&quot;).isNull  //this is coming from groupcontract<br>      //  &amp;&amp; col(&quot;inventories_.inventoryRefId&quot;) == col(&quot;counters_.inventoriesRefID&quot;))  // this is coming from product and offer<br>        &amp;&amp; col(&quot;inventories_.inventoryTypeCode&quot;) == col(&quot;counters_.inventoryTypeCode&quot;))<br>      .withColumn(&quot;verb&quot;, when(col(&quot;actionEnum&quot;) === &quot;Create&quot;, lit(&quot;B&quot;)).when(col(&quot;actionEnum&quot;) === &quot;Update&quot;, lit(&quot;A&quot;)).otherwise(lit(&quot;D&quot;)))<br>      .withColumn(&quot;amount&quot;, lit(null))<br>      .withColumn(&quot;currencyCode&quot;, lit(null))<br>      .withColumn(&quot;dec_pre&quot;, lit(null))<br> //     .withColumn(&quot;blocked&quot;, col(&quot;counters_.deducted&quot;))<br>      .withColumn(&quot;blocked&quot;, col(&quot;counters_.countersPerInventoryType.deducted&quot;))<br>    // Valid Dataframe which has Rates info<br>    val validDataDF = finaldf.filter(col(&quot;rates_.inventoryTypeCode&quot;).isNotNull)<br>      .filter(col(&quot;rates_.inventoryTypeCode&quot;) === col(&quot;inventories_.inventoryTypeCode&quot;)<br>        &amp;&amp;<br>       // col(&quot;inventories_.inventoryRefId&quot;) === col(&quot;counters_.inventoriesRefID&quot;)<br>        col(&quot;inventories_.inventoryTypeCode&quot;) === col(&quot;counters_.inventoryTypeCode&quot;)<br>        &amp;&amp;<br>        col(&quot;inventories_.baseOccupancy&quot;) &gt;= 1)<br>      .withColumn(&quot;verb&quot;, when(col(&quot;actionEnum&quot;) === &quot;Create&quot;, lit(&quot;C&quot;)).when(col(&quot;actionEnum&quot;) === &quot;Update&quot;, lit(&quot;U&quot;)).otherwise(lit(&quot;D&quot;)))<br>      .withColumn(&quot;amount&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;rates_.amountValue.value&quot;)))<br>      .withColumn(&quot;currencyCode&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;rates_.amountValue.currencyCode&quot;)))<br>      .withColumn(&quot;dec_pre&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;rates_.amountValue.numberOfDecimals&quot;)))<br>   //   .withColumn(&quot;blocked&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;counters_.deducted&quot;)))<br>   .withColumn(&quot;blocked&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;counters_.countersPerInventoryType.deducted&quot;)))<br><br>    // Union of both Absense and Valid Dataframes<br>    var dataDF: Dataset[Row] = absenceDataDF<br>      .union(validDataDF)<br><br>    dataDF = dataDF<br>      .withColumn(&quot;HotelId&quot;, concat(col(&quot;propertyCode&quot;), col(&quot;groupCode&quot;), col(&quot;confirmationNumber&quot;)))<br>      .drop(&quot;rates_&quot;, &quot;inventories_&quot;, &quot;counters_&quot;)<br><br>    log.info(<br>      s&quot;Columns of dataDf dataframe before Column map :: ${dataDF.columns.mkString(&quot;Array(&quot;, &quot;, &quot;, &quot;)&quot;)}&quot;,<br>      customerMeta<br>    )<br><br>    // Fetching schema for column mapping<br>   // println(&quot;---xyz---&quot;)<br>    //dataDF.printSchema()<br>    //dataDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, &quot;false&quot;).start()<br>   dataDF<br><br>  }<br>}<br><br><br><br><br><br>groupblockHelper:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.helper<br>import java.sql.{Connection, SQLException, Statement}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants._<br><br>import java.sql.{Connection, Date, DriverManager, ResultSet, Statement, Timestamp}<br>import java.time.temporal.ChronoUnit<br>import org.apache.spark._<br>import com.marriott.elevate.spark.framework.cdc.CUDEvent<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails, MessageBrokerDetails}<br>import com.marriott.elevate.spark.framework.entity.DefaultEventInfo<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br>import com.marriott.elevate.spark.framework.utils.DateUtil._<br>import com.marriott.elevate.spark.framework.utils.{JsonUtil, S3Util, XMLUtil}<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbSink<br>import com.marriott.mgpapp.spark.groupblock.collation.udfs.SparkUdfs._<br>import com.marriott.mgpapp.spark.groupblock.collation.dao.GroupBlockRow<br>import com.marriott.mgpapp.spark.groupblock.Constants._<br>import com.marriott.elevate.spark.framework.cdc.CUDPayload<br>import com.marriott.elevate.spark.framework.cdc.CUDBaseTable<br>import com.marriott.elevate.spark.framework.datasource.PostgresSqlDataSource<br>import com.marriott.mgpapp.spark.groupblock.collation.exceptions.PostgresException<br>import com.marriott.utility.CustomLogger<br>import org.joda.time.DateTime<br>import org.joda.time.format.ISODateTimeFormat<br>import shapeless.syntax.std.tuple.productTupleOps<br><br>import scala.util.{Failure, Success, Try}<br>import org.apache.spark.sql.{Encoder, Encoders}<br><br>trait GroupBlockHelper extends LazyLogging with Serializable {<br><br><br>  val schemaMap: Map[String, String]<br>  val log: CustomLogger<br>  val customerMeta: CustomloggerMetadata<br>  val jobName:String<br>  val jobIdentifier:String<br><br>  // Set of Functions for getting latest and greatest record.<br>  def findLatestRecords(groupedKeyValue: (String, String, String, String, Date, String), records: Iterator[GroupBlockCollationSchema]): GroupBlockCollationSchema = {<br>    val latestRecord = records.reduceLeft((x, y) =&gt; getLatestRecord(x, y))<br>    latestRecord<br><br>  }<br><br>  def getLatestRecord(x: GroupBlockCollationSchema, y: GroupBlockCollationSchema): GroupBlockCollationSchema = {<br>    val latestEvent = if (getLoadTimeStamp(x).isAfter(getLoadTimeStamp(y))) x else y<br>    latestEvent<br>  }<br><br>  def getLoadTimeStamp(event: GroupBlockCollationSchema): DateTime = {<br>    getTS(event.transmissionTimestamp).get<br>  }<br><br>  def getTS(timestamp: String): Option[DateTime] = timestamp match {<br>    case &quot;&quot; =&gt; None<br>    case _ =&gt;<br>      Try(ISODateTimeFormat.dateTimeParser.parseDateTime(timestamp)) match {<br>        case Success(t) =&gt; Some(t)<br>        case Failure(_) =&gt; None<br>      }<br>  }<br><br>  // Function for Calculating the blocked revenue<br>  def calculateBlockedRevenue(gbDataset: Dataset[GroupBlockCollationEvent])<br>                             (implicit jobConfigurationDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails,<br>                              spark: SparkSession,<br>                              dbDetails: DbDetails):Dataset[(GroupBlockRow, String)] = {<br>    gbDataset.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    import spark.implicits._<br>    log.info(&quot;Calculating blocked Revenue for Group Block &quot; + &quot;- calculateBlockedRevenue()&quot;, customerMeta)<br><br>    log.info(&quot;Starting here :: creating key for transform&quot;, customerMeta)<br>    val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br><br>    val reqPropGBDataWithSrcIndDF: Dataset[GroupBlockEntityNew] = getReqPropGBDataWithSrcIndDF(dataGB, connectionConfigDetails, schemaMap)<br><br>    val invDtLevelDataDS = reqPropGBDataWithSrcIndDF.flatMap(x =&gt; {<br>      actDF(x, spark)<br>    })<br><br>    val invDtLevel = invDtLevelDataDS.as[GroupBlockEntityNewUpdated]<br>    val invDtLevelDataWithCurCodeDS = getInvDtLevelDataWithCurCodeDS(invDtLevel, connectionConfigDetails, schemaMap)<br><br>    val finalRevenue = processRevenueData(invDtLevelDataWithCurCodeDS, jobName, jobIdentifier)<br>    finalRevenue<br>  }<br><br>  def transformGroupBlockRecords(key: String, iterator: Iterator[GroupBlockCollationEvent])<br>                                (implicit connectionConfigDetails: ConnectionConfigDetails,<br>                                 sparkSession: SparkSession)<br>  : List[GroupBlockEntity] = {<br><br>    log.info(&quot; Transforming groupBlock Records at :: transformGroupBlockRecords()&quot;, customerMeta)<br><br>    val keyValuePair = getDbLookupPjGb(key, iterator, connectionConfigDetails, schemaMap)<br>    val gbList = mapGroupBlockList(iterator, keyValuePair)<br>    val PhMSTsList = filterPhMSTsList(gbList)<br>    val gbEntity = getGbEntity(PhMSTsList)<br>    val finalGbEntity = processGbEntity(keyValuePair, gbEntity, key, iterator, connectionConfigDetails, schemaMap)<br>    finalGbEntity<br>  }<br><br>  // Function for getting lookup data<br><br><br>  // Function for merging multiple records to single record<br>  def mergeToSingleRecord(row: GroupBlockData, records: Iterator[Row]): (GroupBlockRow, String) = {<br>    val jsonDataString = records.map(row =&gt; row.getAs(GroupBlockEventJson).toString).mkString(&quot;|&quot;)<br>    val groupBlockRow = GroupBlockRow(row.pgb_prop_code, row.pgb_group_code, Some(row.pgb_cnfrm_num), row.pgb_grp_ctrct_ind,<br>      row.pgb_start_dt, row.pgb_end_dt, row.pgb_oppty_id, row.pgb_quote_id, row.pgb_group_name, row.pgb_transmission_ts,<br>      row.pgb_verb, row.pgb_last_up_by, new Timestamp(new java.util.Date().getTime))<br>    (groupBlockRow, jsonDataString)<br>  }<br><br>  def getDbLookupPjGb(key: String, gbCollation: Iterator[GroupBlockCollationEvent], connConfig: ConnectionConfigDetails, schemaMap: Map[String, String]<br>                     ): Map[String, String] = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getDbLookupPjGb() on table - &quot; + schemaMap.get(GroupBlockTable).get, customerMeta)<br><br>    var connection: Connection = null<br>    var statement: Statement = null<br>    var rs: ResultSet = null<br>    Try {<br>      //NYCMQSBC9129751<br>      val propCode = key.substring(0, 5)<br>      val groupCode = key.substring(5, 8)<br>      val confirmationNumber = key.substring(8, key.length)<br>      val query = &quot;SELECT * FROM &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(GroupBlockTable).get) + &quot; where pgb_prop_code=&quot; + &quot;&#39;&quot; + propCode + &quot;&#39;&quot; + &quot; and pgb_group_code=&quot; + &quot;&#39;&quot; + groupCode + &quot;&#39;&quot; + &quot; and pgb_cnfrm_num=&quot; + confirmationNumber<br>      log.info(&quot;Query in getDbLookupPjGb() ===&gt;&quot; + query, customerMeta)<br>      Class.forName(connConfig.dbconfig.get.drivername)<br>      connection = DriverManager.getConnection(connConfig.dbconfig.get.jdbcurl, connConfig.dbconfig.get.username, connConfig.dbconfig.get.password)<br>      statement = connection.createStatement<br>      try {<br>        val rs = statement.execute(query)<br><br>      } catch {<br>        case exception: SQLException =&gt;<br>          new PostgresException(exception, customerMeta).handle()<br>      }<br>      rs = statement.executeQuery(query)<br><br>      if (rs.next()) {<br>        Map(&quot;pgb_transmission_ts&quot; -&gt; rs.getTimestamp(&quot;pgb_transmission_ts&quot;).toString,<br>          &quot;pgb_start_dt&quot; -&gt; rs.getTimestamp(&quot;pgb_start_dt&quot;).toLocalDateTime.toLocalDate.toString,<br>          &quot;pgb_end_dt&quot; -&gt; rs.getTimestamp(&quot;pgb_end_dt&quot;).toLocalDateTime.toLocalDate.toString)<br>      }<br>      else {<br>        Map(&quot;pgb_transmission_ts&quot; -&gt; null)<br>      }<br><br>    } match {<br>      case Success(success) =&gt;<br>        log.info(&quot;Succesfully Retreived from :::&quot; + schemaMap.get(GroupBlockTable).get, customerMeta)<br>        rs.close()<br>        statement.close()<br>        connection.close()<br>        success<br>      case Failure(exception) =&gt;<br>        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + exception.printStackTrace(), customerMeta)<br>        new PostgresException(exception, customerMeta).handle()<br>        Map(&quot;pgb_transmission_ts&quot; -&gt; null)<br>    }<br>  }<br><br>  def getDbLookupBrandCode(key: String, groupBlockEvnts: Iterator[GroupBlockCollationEvent], connConfig: ConnectionConfigDetails, schemaMap: Map[String, String]<br>                          ): Boolean = {<br>    log.info(&quot;Performing DB Lookup at ::: getDbLookupBrandCode() on table - &quot; + schemaMap.get(PropertyTable).get, customerMeta)<br><br>    var connection: Connection = null<br>    var statement: Statement = null<br>    var rs: ResultSet = null<br>    Try {<br>      //NYCMQSBC9129751<br>      val propCode = key.substring(0, 5)<br><br>      val query = &quot;select pt_prop_code, pt_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(PropertyTable).get) + &quot; where pt_prop_code=&quot; + &quot;&#39;&quot; + propCode + &quot;&#39;&quot; + &quot; and pt_brand_code in (select uc_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(UserControlTable).get) + &quot; where uc_brand_oy_cmnty=&#39;Y&#39; and uc_prop_code=&#39;ALL00&#39;)&quot;<br>      log.info(&quot;Query in getDbLookupBrandCode() ===&gt;&quot; + query, customerMeta)<br>      Class.forName(connConfig.dbconfig.get.drivername)<br>      connection = DriverManager.getConnection(connConfig.dbconfig.get.jdbcurl, connConfig.dbconfig.get.username, connConfig.dbconfig.get.password)<br>      statement = connection.createStatement<br>      //Executing first SELECT query<br>      rs = statement.executeQuery(query)<br>      rs.next()<br>    } match {<br>      case Success(success) =&gt;<br>        log.info(&quot;Succesfully Retreived from :::&quot; + schemaMap.get(PropertyTable).get, customerMeta)<br>        rs.close()<br>        statement.close()<br>        connection.close()<br>        success<br>      case Failure(exception) =&gt;<br>        log.error(&quot;Failed to retreive from ::: &quot; + exception.printStackTrace(),<br>          customerMeta)<br>        new PostgresException(exception, customerMeta).handle()<br>        false<br>    }<br>  }<br><br>//  def getConnection(connectionConfigDetails: ConnectionConfigDetails): Option[Connection] = {<br>//    log.info(&quot;getConnection():::&quot;, customerMeta)<br>//<br>//    val connectionResult = Try {<br>//      PostgresSqlDataSource.setConfig(connectionConfigDetails.dbconfig.get)<br>//      PostgresSqlDataSource.getConnection.right.get<br>//    }<br>//    connectionResult match {<br>//      case Success(s) =&gt;<br>//        log.info(&quot;Connection established succesfully&quot;, customerMeta)<br>//        Some(s)<br>//      case Failure(exception) =&gt;<br>//        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + exception.printStackTrace(), customerMeta)<br>//        new PostgresException(exception, customerMeta).handle()<br>//        None<br>//    }<br>//  }<br>//<br>//  def closeComponent(wrapperComponent: AutoCloseable): Unit = {<br>//    val componentName = wrapperComponent.getClass.getSimpleName<br>//    Try(wrapperComponent.close()) match {<br>//      case Success(_) =&gt; // logger.info(s&quot;Closed $componentName Successfully&quot;)<br>//      case Failure(exception) =&gt;<br>//        log.error(&quot;Unable to close ::: &quot; + componentName + &quot;with Exception ::&quot; + exception.printStackTrace(),<br>//          customerMeta)<br>//    }<br>//  }<br><br><br>  def getLookupDataPropParamAssignMap(propCode: String, connConfig: ConnectionConfigDetails, schemaMap: Map[String, String]): Map[String, String] = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getLookupDataPropParamAssignMap() on table - prop_param_assign&quot;, customerMeta)<br><br>    var connection: Connection = null<br>    var statement: Statement = null<br>    var rs: ResultSet = null<br>    val quote = &quot;&#39;&quot;<br>    Try {<br>      val sqlQuery = s&quot;select $PpaPropCode  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTable).get} where $PpaParamId=&#39;DATASRCIND&#39; and $PpaParamValue=&#39;S&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;<br>      log.info(&quot;Query in getLookupDataPropParamAssignMap() ===&gt;&quot; + sqlQuery, customerMeta)<br>      Class.forName(connConfig.dbconfig.get.drivername)<br>      connection = DriverManager.getConnection(connConfig.dbconfig.get.jdbcurl, connConfig.dbconfig.get.username, connConfig.dbconfig.get.password)<br>      statement = connection.createStatement<br>      try {<br>        val rs = statement.execute(sqlQuery)<br><br>      } catch {<br>        case exception: SQLException =&gt;<br>          new PostgresException(exception, customerMeta).handle()<br>      }<br>      rs = statement.executeQuery(sqlQuery)<br><br>      if (rs.next()) {<br>        Map(PpaPropCode -&gt; rs.getString(PpaPropCode))<br>      }<br>      else {<br>        Map.empty[String, String]<br>      }<br><br>    } match {<br>      case Success(success) =&gt;<br>        log.info(&quot;Succesfully Retreived from :::&quot;+ schemaMap.get(PropParamAssignTable).get + success , customerMeta)<br>        rs.close()<br>        statement.close()<br>        connection.close()<br>        success<br>      case Failure(exception) =&gt;<br>        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + exception.printStackTrace(), customerMeta)<br>        new PostgresException(exception, customerMeta).handle()<br>        Map.empty[String, String]<br>    }<br><br>  }<br><br>  def getLookupDataPropParamAssignWithPropcodeAndParamValue(propCode: String, connConfig: ConnectionConfigDetails, schemaMap: Map[String, String]): String = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getLookupDataPropParamAssignWithPropcodeAndParamValue() on table - prop_param_assign&quot;, customerMeta)<br><br>    var connection: Connection = null<br>    var statement: Statement = null<br>    var rs: ResultSet = null<br>    Try {<br>      val sqlQuery = s&quot;select $PpaParamValue  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTable).get} where $PpaParamId=&#39;SELLCURCD&#39; and $PpaActiveInd=&#39;Y&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;<br>      log.info(&quot;Query in getLookupDataPropParamAssignWithPropcodeAndParamValue() ===&gt;&quot; + sqlQuery, customerMeta)<br>      Class.forName(connConfig.dbconfig.get.drivername)<br>      connection = DriverManager.getConnection(connConfig.dbconfig.get.jdbcurl, connConfig.dbconfig.get.username, connConfig.dbconfig.get.password)<br>      statement = connection.createStatement<br>      try {<br>        val rs = statement.execute(sqlQuery)<br><br>      } catch {<br>        case exception: SQLException =&gt;<br>          new PostgresException(exception, customerMeta).handle()<br>      }<br>      rs = statement.executeQuery(sqlQuery)<br><br>      if (rs.next()) rs.getString(&quot;ppa_param_value&quot;)<br>      else EmptyString<br><br>    } match {<br>      case Success(success) =&gt;<br>        log.info(&quot;Succesfully Retreived CurrencyCode from :::&quot; + schemaMap.get(PropParamAssignTable).get + success, customerMeta)<br>        rs.close()<br>        statement.close()<br>        connection.close()<br>        success<br>      case Failure(exception) =&gt;<br>        log.error(&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot; + exception.printStackTrace(), customerMeta)<br>        new PostgresException(exception, customerMeta).handle()<br>        EmptyString<br>    }<br><br>  }<br><br>//  def getLookupDataPropParamAssignWithPropcodeAndParamValue(ppa_prop_code: String, connectionConfigDetails: ConnectionConfigDetails, schemaMap: Map[String, String]): String = {<br>//<br>//    log.info(&quot;Performing DB Lookup at ::: getLookupDataPropParamAssignWithPropcodeAndParamValue() on table - prop_param_assign&quot;, customerMeta)<br>//<br>//    val connection = getConnection(connectionConfigDetails)<br>//    val propParamAssignDs: String =<br>//      connection.isDefined match {<br>//        case true =&gt;<br>//<br>//          val sqlQuery = s&quot;select $PpaParamValue  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTable).get} where $PpaParamId=&#39;SELLCURCD&#39; and $PpaActiveInd=&#39;Y&#39; and  $PpaPropCode = ? &quot;<br>//          log.info(&quot;Query in getLookupDataPropParamAssignWithPropcodeAndParamValue() ===&gt;&quot; + sqlQuery, customerMeta)<br>//          val preparedStatement = connection.get.prepareStatement(sqlQuery)<br>//          preparedStatement.setString(1, ppa_prop_code)<br>//<br>//          val resultSet = preparedStatement.executeQuery<br>//          val currencyCode = if (resultSet.next()) {<br>//            resultSet.getString(1)<br>//          } else EmptyString<br>//<br>//          closeComponent(resultSet)<br>//          closeComponent(preparedStatement)<br>//          log.info(&quot; getLookupDataPropParamAssignWithPropcodeAndParamValue propParamAssignRecords :: &quot; + currencyCode, customerMeta)<br>//          currencyCode<br>//<br>//        case false =&gt;<br>//          log.error(&quot;GroupBlock job error getLookupDataPropParamAssignWithPropcodeAndParamValue &quot;, customerMeta)<br>//<br>//          EmptyString<br>//      }<br>//    connection match {<br>//      case Some(dbConnection) =&gt; closeComponent(dbConnection)<br>//      case None =&gt; //  logger.info(&quot;DB Connect Closed&quot;)<br>//    }<br>//    propParamAssignDs<br>//  }<br><br>  def summarizeDetail(groupByKey: (String, String, String, Date, Date,<br>    String, String, String, String, String, String, String, String, Date, String, Option[Int]),<br>                      rec: Iterator[GroupBlockEntityUpdatedCurrencyCode]): GroupBlockCollationSchema = {<br><br>    val records = rec.toSeq<br>    //ELEV-84286 fix<br>    //for ratePlan PPMS the rateAmount is coming as 0 adding so considering record as is.<br>    //Handling multiple ratePlan with multiple rates defined for single inventoryTypeCode<br>    val recordsWithNonZeroAmt = records.filter(x =&gt; x.Amt &gt; 0 &amp;&amp; (x.inventoryDate.after(x.ratePlanEffectiveDate) || x.inventoryDate.equals(x.ratePlanEffectiveDate)) &amp;&amp; x.inventoryDate.before(x.ratePlanExpireDate))<br>    val filteredRecords = if (recordsWithNonZeroAmt.isEmpty) {<br>      records<br>    }<br>    else {<br>      recordsWithNonZeroAmt<br>    }<br>    val totalAmt = filteredRecords.map(f =&gt; f.Amt).sum<br>    val recordCount = filteredRecords.size<br>    val avg = totalAmt / recordCount<br>    //val blockedRevenue = avg * recordCount * recordsWithNonZeroAmt.head.blockedRooms.getOrElse(0)<br>    //removing records count from here<br>    val blockedRevenue = avg * filteredRecords.head.blockedRooms.getOrElse(0)<br>    val record = filteredRecords.head<br>    val recordTimestamp = record.transmissionTimestamp.replace(&quot;T&quot;, &quot; &quot;).replace(&quot;Z&quot;, &quot;&quot;)<br>    GroupBlockCollationSchema(record.groupCode, record.propertyCode, record.confirmationNumber, record.groupEffectiveDate, record.groupExpireDate,<br>      record.groupIndicator, record.groupSourceIndicator, record.opportunityId, record.quoteId, record.groupEventName, record.transmissionTimestamp,<br>      record.currencyCode, record.actionIndicator, record.inventoryDate, record.inventoryTypeCode, record.blockedRooms.getOrElse(0), avg, recordCount, recordTimestamp, blockedRevenue)<br>  }<br><br><br><br>  /**<br>   * Parse the default event info intoJson format<br>   *<br>   * @param entity<br>   */<br><br>  def actDF(actualDF:GroupBlockEntityNew,spark:SparkSession):Seq[GroupBlockEntityNewUpdated]= {<br>    import spark.implicits._<br>    val groupBlockList = if (actualDF.InvDateSpan &gt; 0) {<br>      for {i &lt;- 1 to actualDF.InvDateSpan} yield {<br>        val GBEntity = GroupBlockEntityNewUpdated(<br>          actualDF.actionIndicator,<br>          actualDF.transmissionTimestamp,<br>          actualDF.opportunityId,<br>          actualDF.quoteId,<br>          actualDF.confirmationNumber,<br>          actualDF.propertyCode,<br>          actualDF.groupCode,<br>          actualDF.groupEffectiveDate,<br>          actualDF.groupExpireDate,<br>          actualDF.groupEventName,<br>          actualDF.groupEventType,<br>          actualDF.inventoryTypeCode,<br>          actualDF.ratePlanCode,<br>          actualDF.ratePlanEffectiveDate,<br>          actualDF.ratePlanExpireDate,<br>          actualDF.amount,<br>          actualDF.currencyCode,<br>          actualDF.decimalPrecision,<br>          actualDF.blockedRooms,<br>          actualDF.inventoryEffectiveDate,<br>          actualDF.inventoryExpireDate,<br>          actualDF.ppa_prop_code,<br>          actualDF.data_source_ind,<br>          actualDF.groupIndicator,<br>          actualDF.groupSourceIndicator,<br>          // actualDF.InvDateSpan,<br>          // actualDF.InventoryDateArray,<br>          actualDF.inventoryDate,<br>          actualDF.Amt<br>          /*if(actualDF.actionIndicator != &quot;A&quot; &amp;&amp; actualDF.actionIndicator != &quot;B&quot;) {<br>         if (actualDF.actionIndicator == NotRateRecsIndUpdate) &quot;U&quot;<br>              else if (actualDF.actionIndicator == NotRateRecsIndCreate) &quot;C&quot;<br>              else &quot;D&quot;<br>        } else actualDF.actionIndicator*/<br>        )<br>        GBEntity<br>      }<br>    } else {<br>      Seq(GroupBlockEntityNewUpdated(<br>        actualDF.actionIndicator,<br>        actualDF.transmissionTimestamp,<br>        actualDF.opportunityId,<br>        actualDF.quoteId,<br>        actualDF.confirmationNumber,<br>        actualDF.propertyCode,<br>        actualDF.groupCode,<br>        actualDF.groupEffectiveDate,<br>        actualDF.groupExpireDate,<br>        actualDF.groupEventName,<br>        actualDF.groupEventType,<br>        actualDF.inventoryTypeCode,<br>        actualDF.ratePlanCode,<br>        actualDF.ratePlanEffectiveDate,<br>        actualDF.ratePlanExpireDate,<br>        actualDF.amount,<br>        actualDF.currencyCode,<br>        actualDF.decimalPrecision,<br>        actualDF.blockedRooms,<br>        actualDF.inventoryEffectiveDate,<br>        actualDF.inventoryExpireDate,<br>        actualDF.ppa_prop_code,<br>        actualDF.data_source_ind,<br>        actualDF.groupIndicator,<br>        actualDF.groupSourceIndicator,<br>        // actualDF.InvDateSpan,<br>        // actualDF.InventoryDateArray,<br>        actualDF.inventoryDate,<br>        actualDF.Amt<br>        /*if(actualDF.actionIndicator != &quot;A&quot; &amp;&amp; actualDF.actionIndicator != &quot;B&quot;) {<br>       if (actualDF.actionIndicator == NotRateRecsIndUpdate) &quot;U&quot;<br>            else if (actualDF.actionIndicator == NotRateRecsIndCreate) &quot;C&quot;<br>            else &quot;D&quot;<br>      } else actualDF.actionIndicator*/<br>      ))<br>    }<br>    groupBlockList.distinct<br>  }<br><br><br>  def checkForNull(str: Any): String = if (str == null) &quot;&quot; else str.toString<br><br>  def mapGroupBlockList(iterator: Iterator[GroupBlockCollationEvent], keyValuePair: Map[String, String]): List[GroupBlockCollationEvent] = {<br>    iterator.toList.map { event =&gt;<br>      val gbCollEvent = event<br>      val cudpLoad = gbCollEvent.getPayload.groupBlockCUD.copy(columnMap = gbCollEvent.getPayload.groupBlockCUD.columnMap ++ keyValuePair)<br>      val gCollationPayLoad = gbCollEvent.groupBlockPayload.copy(groupBlockCUD = cudpLoad)<br>      gbCollEvent.copy(groupBlockPayload = gCollationPayLoad)<br>    }<br>  }<br><br>  def filterPhMSTsList(gbList: List[GroupBlockCollationEvent]): List[GroupBlockCollationEvent] = {<br>    gbList.filter { x =&gt;<br>      val loadTs = x.getPayload.groupBlockCUD.baseTable.loadTimeStamp<br>      val phDbTs = x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;pgb_transmission_ts&quot;, NULL)<br>      if (phDbTs == NULL) true else getTimestamp(loadTs, UtcTimestampFormat).get.after(getTimestamp(phDbTs).get)<br>    }<br>  }<br><br>  def getGbEntity(PhMSTsList: List[GroupBlockCollationEvent]): List[GroupBlockEntity] = {<br>    PhMSTsList.flatMap(x =&gt; {<br>      val gBEvents = if (x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == Delete) {<br>        Set(GroupBlockEntity(<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          None, None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          None, None,<br>          None, None, None, None, None, None, None, None, None, None, None))<br>      }<br>      else if ((x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == UdateRateAbsence) || (x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == CreateRateAbsence)) {<br>        log.info(&quot; Rate Absence Records:: InventoryTypeCode = &quot; + x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;), customerMeta)<br>        Set(GroupBlockEntity(Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;opportunityID&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;quoteId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupName&quot;, &quot;&quot;).toString()),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupType&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;)),<br>          None, None, None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;amount&quot;, &quot;&quot;).toInt),<br>          None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;dec_pre&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;blocked&quot;, &quot;&quot;).toInt),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryExpireDate&quot;, &quot;&quot;), DATE_FORMAT)))<br>      }<br>      else {<br>        Set(GroupBlockEntity(Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;opportunityID&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;quoteId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupName&quot;, &quot;&quot;).toString()),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupType&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;amount&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;currencyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;dec_pre&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;blocked&quot;, &quot;&quot;).toInt),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryExpireDate&quot;, &quot;&quot;), DATE_FORMAT)))<br>      }<br><br>      /* Added below logic to include &#39;CMP&#39; groupBlock which has rooms but no revenue for the InventoryTypeCode.<br>              Updating actionIndicator from A to U*/<br>      val groupBlockUpdatedEvents = gBEvents.toList<br>        .groupBy(x =&gt; (x.groupCode, x.propertyCode)).flatMap { case (key, events) =&gt; {<br><br>          if (key._1.get.equalsIgnoreCase(&quot;CMP&quot;)) {<br>            val UpdateRateAbsence = events.filter(_.actionIndicator.get.equals(&quot;A&quot;))<br>            val UpdateEvents = events.filter(_.actionIndicator.get.equals(&quot;U&quot;))<br><br>            if (UpdateEvents.isEmpty &amp;&amp; UpdateRateAbsence.nonEmpty) {<br>              List(UpdateRateAbsence.head.copy(actionIndicator = Some(&quot;U&quot;)))<br>            } else events<br>          } else events<br>        }<br>        }<br>      // logger.info(s&quot;groupBlockUpdatedEvents :: $groupBlockUpdatedEvents&quot;)<br><br>      groupBlockUpdatedEvents<br><br>    }).filter(_.actionIndicator != UdateRateAbsence).filter(_.actionIndicator != CreateRateAbsence)<br>  }<br><br>  def processGbEntity(keyValuePair: Map[String, String], gbEntity: List[GroupBlockEntity], key: String, iterator: Iterator[GroupBlockCollationEvent],<br>                      connectionConfigDetails: ConnectionConfigDetails, schemaMap: Map[String, String])<br>                     (implicit sparkSession: SparkSession): List[GroupBlockEntity] = {<br>    if (keyValuePair(&quot;pgb_transmission_ts&quot;) != NULL &amp;&amp; gbEntity.nonEmpty &amp;&amp; !gbEntity.head.actionIndicator.contains(Delete)<br>      &amp;&amp; !gbEntity.head.actionIndicator.contains(UdateRateAbsence)<br>      &amp;&amp; !gbEntity.head.actionIndicator.contains(CreateRateAbsence)) {<br><br>      val requiredFields = gbEntity.filterNot(x =&gt; x.actionIndicator.contains(Delete)).head<br><br>      val (leftStartDate, leftEndDate) = getLeftDates(keyValuePair, requiredFields)<br>      val (rightStartDate, rightEndDate) = getRightDates(keyValuePair, requiredFields)<br><br>      val initialDeleteSeq = createDeleteSeq(requiredFields, leftStartDate, leftEndDate)<br>      val laterDeleteSeq = createDeleteSeq(requiredFields, rightStartDate, rightEndDate)<br><br>      val requiredGBEntity = initialDeleteSeq ++ laterDeleteSeq ++ gbEntity<br><br>      logger.info(&quot;Required entity&quot;, requiredGBEntity.toString())<br>      val lookupSuccess = getDbLookupBrandCode(key, iterator, connectionConfigDetails, schemaMap)<br><br>      if (lookupSuccess) {<br>        val DeletedEvents = requiredGBEntity.toSet.toList.filter(x =&gt; if (x.actionIndicator.get == &quot;D&quot;) true else false)<br>        if (DeletedEvents.size &gt; 0) {<br>          DeletedEvents.toSet.toList<br>        }<br>        else {<br>          requiredGBEntity.toSet.toList<br>        }<br>      }<br>      else List()<br>    }<br>    else {<br>      val lookupSuccess = getDbLookupBrandCode(key, iterator, connectionConfigDetails, schemaMap)<br><br>      val DeletedEvents = gbEntity.toSet.toList.filter(x =&gt; if (x.actionIndicator.get == &quot;D&quot;) true else false)<br><br>      if (lookupSuccess) {<br>        if (DeletedEvents.size &gt; 0) {<br>          DeletedEvents.toSet.toList<br>        }<br>        else gbEntity.toSet.toList<br>      }<br>      else List()<br>    }<br>  }<br><br>  def getLeftDates(keyValuePair: Map[String, String], requiredFields: GroupBlockEntity): (Date, Date) = {<br>    if (Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)).before(requiredFields.groupEffectiveDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))<br>    }<br>    else if (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)).before(requiredFields.groupEffectiveDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), requiredFields.groupEffectiveDate.get)}<br>    else {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)))}}<br><br>  def getRightDates(keyValuePair: Map[String, String], requiredFields: GroupBlockEntity): (Date, Date) = {<br>    if (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)).after(requiredFields.groupExpireDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))<br>    }<br>    else if (requiredFields.groupExpireDate.get.before(Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))) {<br>      (requiredFields.groupExpireDate.get, Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))}<br>    else {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)))}}<br><br>  def createDeleteSeq(requiredFields: GroupBlockEntity, startDate: Date, endDate: Date): List[GroupBlockEntity] = {<br>    List(GroupBlockEntity(Some(&quot;DS&quot;), requiredFields.transmissionTimestamp, None, None,<br>      requiredFields.confirmationNumber, requiredFields.propertyCode, requiredFields.groupCode, Some(startDate),<br>      Some(endDate), None, Some(&quot;CONVENTION&quot;), Some(&quot;XXX&quot;), None, Some(startDate), Some(endDate), Some(1),<br>      Some(&quot;XXX&quot;), None, Some(1), Some(startDate), Some(endDate)) )<br>      .filterNot(x =&gt; x.groupEffectiveDate == x.groupExpireDate)<br>  }<br><br>  def getReqPropGBDataWithSrcIndDF(dataGB: Dataset[GroupBlockEntity], connectionConfigDetails: ConnectionConfigDetails, schemaMap: Map[String, String])(implicit spark: SparkSession): Dataset[GroupBlockEntityNew] = {<br>    import spark.implicits._<br><br>    log.info(&quot;getReqPropGBDataWithSrcIndDF&quot;,customerMeta)<br>    dataGB.groupByKey(x =&gt; (x.propertyCode.get))<br>      .mapGroups { case (rowid, event) =&gt; event.toList }.flatMap(record =&gt; {<br>        val ppa_prop_code = record.head.propertyCode.get<br>        val reqPropGBData = getLookupDataPropParamAssignMap(ppa_prop_code, connectionConfigDetails, schemaMap)<br>        record.map(record =&gt; {<br>          val GBParamAssignDS = GroupBlockEntityPropCode(record.actionIndicator, record.transmissionTimestamp,<br>            record.opportunityId, record.quoteId, record.confirmationNumber, record.propertyCode, record.groupCode,<br>            record.groupEffectiveDate, record.groupExpireDate, record.groupEventName, record.groupEventType,<br>            record.inventoryTypeCode, record.ratePlanCode, record.ratePlanEffectiveDate, record.ratePlanExpireDate,<br>            record.amount, record.currencyCode, record.decimalPrecision, record.blockedRooms,<br>            record.inventoryEffectiveDate, record.inventoryExpireDate,<br>            reqPropGBData.getOrElse(PpaPropCode, NULL))<br>          GBParamAssignDS<br>        })<br>      }).toDF().withColumn(DataSourceInd, when(trim(col(<br>        PropertyCodeConst)) === trim(col(PpaPropCode)), lit(SgdsInd))<br>        .otherwise(lit(NonSgdsInd)))<br>      .withColumn(GroupIndicator, when(upper(col(GroupEventType)) === lit(Convention), GroupInd)<br>        .when(upper(col(GroupEventType)) === lit(Contract), ContractInd))<br>      .withColumn(GroupSrcIndicator, when(upper(col(GroupEventType)) === lit(Convention) and<br>        upper(col(DataSourceInd)) === lit(SgdsInd), lit(SgdsInd))<br>        .when(upper(col(GroupEventType)) === lit(Convention) and<br>          upper(col(DataSourceInd)) === lit(NonSgdsInd), lit(NonSgdsInd))<br>        .when(upper(col(GroupEventType)) === lit(Contract), lit(ContractInd)))<br>      .withColumn(InvDateSpan, lit(0))<br>      .withColumn(InvDate, when(col(&quot;InventoryEffectiveDate&quot;).isNotNull, col(&quot;InventoryEffectiveDate&quot;)).otherwise(when(col(&quot;ActionIndicator&quot;) =!= &quot;D&quot;, current_date()).otherwise(to_date(lit(&quot;9999-12-31&quot;), &quot;yyyy-MM-dd&quot;))))<br>      .withColumn(Amt, getAmount(when(col(Amount).isNotNull, col(Amount)).otherwise(0),<br>        when(col(DecPrec).isNotNull, col(DecPrec)).otherwise(0)))<br>      .withColumn(&quot;AdditionalIndicator&quot;, when((col(ActionIndicator) === &quot;A&quot; &amp;&amp; col(BlockedRooms).eq(Some(0)))<br>        || (col(ActionIndicator) === &quot;B&quot; &amp;&amp; col(BlockedRooms).eq(Some(0))), lit(0)).otherwise(1))<br>      .as[GroupBlockEntityAdj]<br>      .filter(col(&quot;AdditionalIndicator&quot;) === 1)<br>      .drop(col(&quot;AdditionalIndicator&quot;))<br>      .as[GroupBlockEntityNew]<br>  }<br><br>  def getInvDtLevelDataWithCurCodeDS(invDtLevel: Dataset[GroupBlockEntityNewUpdated], connectionConfigDetails: ConnectionConfigDetails, schemaMap: Map[String,String])(implicit spark: SparkSession): Dataset[GroupBlockEntityUpdatedCurrencyCode] = {<br>    log.info(&quot;getInvDtLevelDataWithCurCodeDS()&quot;,customerMeta)<br>    import spark.implicits._<br>    invDtLevel.groupByKey(x =&gt; (x.propertyCode))<br>      .mapGroups { case (_, event) =&gt; event.toList }.flatMap(records =&gt; {<br>        val propertyCode = records.head.propertyCode<br>        val currencyCode = getLookupDataPropParamAssignWithPropcodeAndParamValue(propertyCode, connectionConfigDetails, schemaMap)<br>        log.info(&quot;currencyCode ::&quot; + currencyCode, customerMeta)<br>        records.map(record =&gt; {<br>          log.info(&quot;Record currencyCode ::&quot; + {<br>            record.currencyCode<br>          }, customerMeta)<br>          val GroupBlockEntityUpdatedCurrencyCodeRecords = GroupBlockEntityUpdatedCurrencyCode(record.actionIndicator,<br>            record.transmissionTimestamp, record.opportunityId, record.quoteId, record.confirmationNumber,<br>            record.propertyCode, record.groupCode, record.groupEffectiveDate,<br>            record.groupExpireDate, record.groupEventName, record.groupEventType, record.inventoryTypeCode,<br>            record.ratePlanCode, record.ratePlanEffectiveDate, record.ratePlanExpireDate,<br>            record.amount, if (record.currencyCode != null) record.currencyCode else currencyCode, record.decimalPrecision, record.blockedRooms,<br>            record.inventoryEffectiveDate, record.inventoryExpireDate, record.ppa_prop_code,<br>            record.data_source_ind, record.groupIndicator, record.groupSourceIndicator, record.inventoryDate, record.Amt)<br><br>          log.info(&quot;GroupBlockEntityUpdatedCurrencyCodeRecords :: &quot; + GroupBlockEntityUpdatedCurrencyCodeRecords, customerMeta)<br>          GroupBlockEntityUpdatedCurrencyCodeRecords<br>        })<br>      })<br>  }<br><br>  def processRevenueData(invDtLevelDataWithCurCodeDS: Dataset[GroupBlockEntityUpdatedCurrencyCode], jobName: String, jobIdentifier: String)(implicit spark: SparkSession): Dataset[(GroupBlockRow,String)] = {<br>    import spark.implicits._<br>    val avgBlockedActualRevenueDS: Dataset[GroupBlockCollationSchema] = invDtLevelDataWithCurCodeDS<br>      .groupByKey(record =&gt; (record.groupCode, record.propertyCode, record.confirmationNumber, record.groupEffectiveDate, record.groupExpireDate,<br>        record.groupIndicator, record.groupSourceIndicator, record.opportunityId, record.quoteId, record.groupEventName, record.transmissionTimestamp,<br>        record.currencyCode, record.actionIndicator, record.inventoryDate, record.inventoryTypeCode, record.blockedRooms))<br>      .mapGroups(summarizeDetail)<br><br>    val getLatestRecsDS: Dataset[GroupBlockCollationSchema] = avgBlockedActualRevenueDS.as[GroupBlockCollationSchema]<br>      .groupByKey(record =&gt; (record.propertyCode, record.groupCode, record.confirmationNumber, record.quoteId, record.inventoryDate, record.inventoryTypeCode))<br>      .mapGroups(findLatestRecords)<br><br>    val avgBlockedRevenueDS_A: Dataset[(GroupBlockRow,String)] = getLatestRecsDS<br>      .withColumn(GroupBlock,<br>        to_json(struct(ActionIndicator, PropertyCodeConst, GroupCode, RecordTimestamp,<br>          ConfirmationNumber, GroupIndicator, InvDate, InventoryTypeCode, BlockedRooms,<br>          BlockedRevenue, CurrencyCode)))<br>      .withColumn(EventSource, lit(jobName))<br>      .withColumn(EventVersion, lit(EmptyStr))<br>      .withColumn(EventIdentifier, lit(jobIdentifier))<br>      .withColumn(RetryCounter, lit(0))<br>      .withColumn(MetaData, typedLit(Map[String, String]()))<br>      .withColumn(&quot;DestinationTopic&quot;, lit(EmptyStr))<br>      .withColumn(RecordNum, lit(0))<br>      .withColumn(TimeStamp, current_timestamp())<br>      .withColumn(Error, lit(EmptyStr))<br>      .withColumn(ErrorType, EmptyArray())<br>      .withColumn(Message, EmptyArray())<br>      .withColumn(Warning, EmptyArray())<br>      .withColumn(PayLoad, col(GroupBlock))<br>      .withColumn(PayLoadType, lit(Json))<br>      .withColumn(GroupBlockEventJson,<br>        to_json(struct(TimeStamp, MetaData, RetryCounter, PayLoad, EventVersion, EventSource, &quot;DestinationTopic&quot;, EventIdentifier, PayLoadType,<br>          RecordNum, Error, ErrorType, Message, Warning)))<br>      .withColumn(GroupEffDate, when(col(ActionIndicator) === Delete, current_date).otherwise(col(GroupEffectiveDate)))<br>      .withColumn(GroupExpDate, when(col(ActionIndicator) === Delete, current_date).otherwise(col(GroupExpireDate)))<br>      .groupByKey(row =&gt; {<br>        GroupBlockData(<br>          row.getAs(PropertyCodeConst),<br>          row.getAs(GroupCode),<br>          row.getAs[String](ConfirmationNumber).toInt,<br>          checkForNull(row.getAs(GroupIndi)),<br>          Some(row.getAs(GroupEffDate)),<br>          Some(row.getAs(GroupExpDate)),<br>          checkForNull(row.getAs(OpportunityId)),<br>          checkForNull(row.getAs(QuoteId)),<br>          checkForNull(row.getAs(GroupEventName)),<br>          getTimestamp(row.getAs(RecordTimestamp), TimestampFormat),<br>          row.getAs(ActionIndicator),<br>          PgbLastUpdBy)<br>      })<br>      .mapGroups(mergeToSingleRecord)<br>      .map(record =&gt; {<br>        log.info(&quot;avgBlockedRevenueDS :: &quot; + record, customerMeta)<br>        record<br>      })<br><br>    avgBlockedRevenueDS_A<br>  }<br><br><br>}<br><br>jobs<br><br>collation job<br>package com.marriott.mgpapp.spark.groupblock.collation.jobs<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants._<br>import com.marriott.elevate.spark.framework.traits.StreamingJob<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.groupblock.Constants.{CheckpointLocation, Default, GroupBlockCollationDBCheckpointDir, GroupBlockCollationKafkaCheckpointDir, GroupBlockTable, JobIdentifier, JobName, appNameForLogger, checkpointDir, loggerAppName}<br>import com.marriott.mgpapp.spark.groupblock.collation.dao.GroupBlockRow<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbSink<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.{CustomloggerMetadata, ReservationGroupBlockMerge}<br>import com.marriott.mgpapp.spark.groupblock.collation.exceptions.KafkaException<br>import com.marriott.mgpapp.spark.groupblock.collation.helper.{GroupBlockFiltrationHelper, GroupBlockHelper}<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.util._<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql.functions.{col, from_json, lit}<br>import org.apache.spark.sql.streaming.{StreamingQuery, Trigger}<br>import org.apache.spark.sql.types._<br>import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}<br><br>import scala.util.{Failure, Success, Try}<br><br>class GroupBlockCollationJob(implicit<br>                             sparkSession: SparkSession,<br>                             jobConfigDetails: JobConfigurationDetails,<br>                             connectionConfigDetails: ConnectionConfigDetails<br>                            ) extends GroupBlockHelper<br>  with GroupBlockFiltrationHelper<br>  with StreamingJob<br>  with Serializable {<br><br>  this: GroupBlockHelper =&gt;<br><br>  override val kafkaTopicMap: Option[Map[String, String]] = jobConfigDetails.kafkatopicmap<br><br>  override val jobName: String = jobConfigDetails.jobname<br><br>  override val jobIdentifier: String = jobConfigDetails.jobidentifier<br><br>  val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br><br>  implicit val dbDetails: DbDetails = connectionConfigDetails.dbconfig.get<br><br>  private val s3bucket: String = connectionConfigDetails.bucketconfig.get(defaultBucket)<br><br>  override val checkPointDirectory: Map[String, String] =<br>    jobConfigDetails.checkpointdirectory.get<br><br>  implicit val schemaMap: Map[String, String] =<br>    jobConfigDetails.schemamap.get ++ connectionConfigDetails.schemaconfig.get<br><br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br><br>  override val customerMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, map)<br><br>  val spark: SparkSession = SparkSession.builder()<br>    .appName(&quot;Read JSON File&quot;)<br>    .getOrCreate()<br><br>  override def process(args: Map[String, String]): Unit = {<br><br>    log.info(&quot;Entered into code&quot;, customerMeta)<br><br>    // Consuming data from EEH Kafka<br>    val data = consumeDataFromEEH(InputType.Avro)<br>   //data.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, &quot;false&quot;).start()<br><br><br>    val filterDS = groupBlockFiltration(data)<br><br>//    filterDS.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, &quot;false&quot;).start()<br>    val groupBlockDS = calculateBlockedRevenue(filterDS)<br><br><br>    processBatchData(groupBlockDS,dbDetails,schemaMap,jobIdentifier,jobName,avroSchema,spark)<br><br>    spark.streams.awaitAnyTermination()<br><br>  }<br><br>  def consumeDataFromEEH(<br>                          inputType: InputType.InputType<br>                        )(implicit<br>                          sparkSession: SparkSession,<br>                          jobConfigDetails: JobConfigurationDetails<br>                        ): DataFrame = {<br>    val jobDetails = Map(<br>      JobIdentifier -&gt; jobIdentifier,<br>      JobName -&gt; jobName<br>    )<br><br>    val result: Try[DataFrame] = Try {<br>      val sourceTopic = KafkaProperties.sourceTopic<br>//      val startingOffset: String = &quot;&quot;&quot;{&quot;commerce.reservation.group-block.sync&quot;:{&quot;0&quot;:177,&quot;1&quot;:48506,&quot;2&quot;:489,&quot;3&quot;:118}}&quot;&quot;&quot;<br>//val startingOffset: String = &quot;&quot;&quot;{&quot;commerce.reservation.group-block.sync&quot;:{&quot;0&quot;:290,&quot;1&quot;:48520,&quot;2&quot;:493,&quot;3&quot;:272}}&quot;&quot;&quot;<br>     val startingOffset: String = &quot;earliest&quot;<br>      val input = inputType<br><br>      val readStreamData = StreamConsumerHelper.readStreamWithParams(<br>        sparkSession,<br>        sourceTopic,<br>        startingOffset,<br>        inputType = input,<br>        deserializer = Deserializer.deserializerWithUdf<br>      )<br><br>      readStreamData.select(col = &quot;value&quot;)<br>    }<br><br>    result match {<br>      case Success(value) =&gt;<br>        log.info(<br>          &quot;Consumed data from kafka EEH &quot;,<br>          customerMeta<br>        )<br>        value<br>      case Failure(exception) =&gt;<br>        new KafkaException(exception).handle(<br>          jobDetails<br>        )<br>        sparkSession.emptyDataFrame<br>    }<br>  }<br><br>  def processBatchData(groupBlockDS: Dataset[(GroupBlockRow, String)],<br>                       dbDetails: DbDetails,<br>                       schemaMap: Map[String, String],<br>                       jobIdentifier: String,<br>                       jobName: String,<br>                       avroSchema: StructType,<br>                       spark: SparkSession): Unit = {<br>    log.info(&quot;ProcessBatchData()&quot;, customerMeta)<br><br><br>    import spark.implicits._<br><br>    try {<br>      val query = groupBlockDS<br>        .writeStream<br>        .outputMode(&quot;update&quot;)<br>        .foreachBatch((batchDS: org.apache.spark.sql.Dataset[(com.marriott.mgpapp.spark.groupblock.collation.dao.GroupBlockRow, String)], _: Long) =&gt; {<br>          batchDS.persist()<br>          batchDS.foreach { data =&gt; {<br>            val dbSink = new DbSink(dbDetails, schemaMap(SchemaName), schemaMap(GroupBlockTable), jobIdentifier, jobName)<br>            dbSink.open(1.toLong, 2)<br>            dbSink.process(data)<br>          }<br>          }<br>          import spark.implicits._<br>          val jsonStringDataset: Dataset[String] = batchDS.map(_._2)<br>          val flattenedJsonRecordsDataset = jsonStringDataset.flatMap(jsonString =&gt; jsonString.split(&quot;\\|&quot;))<br>          val jsonDF = flattenedJsonRecordsDataset.select(from_json(col(&quot;value&quot;), avroSchema).alias(&quot;value&quot;))<br>          val dataDF = jsonDF.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>            .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>            .withColumn(&quot;timestampType&quot;, lit(0))<br>          dataDF.show()<br>          produceDataToEEH(dataDF)<br>        }<br>        )<br>                    .option(CheckpointLocation, &quot;C:\\Checkpoint\\GBC&quot;)<br>//        .option(CheckpointLocation, S3Util.getCheckPointDirectory(s3bucket, jobName, checkPointDirectory(GroupBlockCollationDBCheckpointDir)))<br>        .trigger(Trigger.ProcessingTime(6000))<br>        .start()<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Failed processing batch&quot; + exception.printStackTrace(), customerMeta)<br>    }<br>  }<br><br><br>  def produceDataToEEH(data: DataFrame): Unit = {<br>    val jobDetails = Map(JobIdentifier -&gt; jobIdentifier, JobName -&gt; jobName)<br>    val destinationTopic = KafkaProperties.destinationTopic<br>    val input = InputType.Avro<br>             val checkPointPathKafka: Option[String] = Option(&quot;C:/Checkpoint/Bucket&quot;)<br>//    val checkPointPathKafka: Option[String] = Some(S3Util.getCheckPointDirectory(s3bucket, jobName, checkPointDirectory(GroupBlockCollationKafkaCheckpointDir)))<br>    val schemapath: String = &quot;ReservationGroupBlockMerge0.1.avsc&quot;<br>    try {<br>      val writeStreamData = StreamProducerHelper.writeWithParam(<br>        data, destinationTopic,<br>        inputType = input,<br>        filePath = Some(schemapath),<br>        caseClass = Option(classOf[ReservationGroupBlockMerge]),<br>        checkPointPath = checkPointPathKafka,<br>        //      s3BackupPath = Option(&quot;C:/Checkpoint/Bucket&quot;),<br>        serializer = Serializer.serializerWithAvroCloudEvent<br>      )<br>      log.info(&quot;Pushed into kafka topic&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, map))<br>    } catch {<br>      case exception: Exception =&gt;<br>        new KafkaException(exception).handle(jobDetails)<br>    }<br>  }<br><br>  val avroSchema = new StructType()<br>    .add(&quot;timeStamp&quot;, StringType, nullable = true)<br>    .add(&quot;metadata&quot;, MapType(StringType, StringType, valueContainsNull = true))<br>    .add(&quot;retryCounter&quot;, IntegerType, nullable = true)<br>    .add(&quot;payLoad&quot;, StringType, nullable = true)<br>    .add(&quot;eventVersion&quot;, StringType, nullable = true)<br>    .add(&quot;eventSource&quot;, StringType, nullable = true)<br>    .add(&quot;destinationTopic&quot;, StringType, nullable = true)<br>    .add(&quot;eventIdentifier&quot;, StringType, nullable = true)<br>    .add(&quot;payLoadType&quot;, StringType, nullable = true)<br>    .add(&quot;recordNum&quot;, StringType, nullable = true)<br>    .add(&quot;error&quot;, StringType, nullable = true)<br>    .add(&quot;errorType&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;message&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;warning&quot;, ArrayType(StringType, containsNull = true))<br>}<br><br><br><br>main<br><br><br>package com.marriott.mgpapp.spark.groupblock.collation.jobs<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.jobs.MgpApp<br>import com.marriott.mgpapp.spark.groupblock.collation.jobs.Main.groupBlockCollationJob<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.SparkConf<br><br>object Main extends MgpApp {<br><br>  implicit val jobConfig:JobConfigurationDetails = jobConfigDetails<br><br>  implicit val connConfig:ConnectionConfigDetails = connectionConfigDetails<br><br>  override def getSparkSession: Option[SparkConf] = Some(<br>    new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;Group Block Collation&quot;).setMaster(&quot;local&quot;)<br>  )<br><br>  val groupBlockCollationJob = new GroupBlockCollationJob()<br>  groupBlockCollationJob.start()<br>}<br><br><br>udfs<br><br>package.scala<br>package com.marriott.mgpapp.spark.groupblock<br><br>package object Constants {<br>  val User = &quot;user&quot;<br>  val Password_Const = &quot;password&quot;<br>  val Driver = &quot;driver&quot;<br>  val appNameForLogger = &quot;Group Block Collation&quot;<br>  val GroupBlockTable = &quot;GroupBlockTable&quot;<br>  val PropertyTable = &quot;PropertyTable&quot;<br>  val UserControlTable = &quot;UserControlTable&quot;<br>  val PropParamAssignTable =<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>helper_new<br>groupblockFiltration:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.helper<br><br>import com.fasterxml.jackson.databind.DeserializationFeature<br>import com.marriott.elevate.spark.framework.cdc.{CUDBaseTable, CUDPayload}<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.elevate.spark.framework.utils.{JsonUtil, S3Util}<br>import com.marriott.mgpapp.spark.groupblock.Constants.{Default, GroupBlockCollationIrrecoverableEntityCheckpointDir, IrrecoverableErrorTopic, IrrecoverableFilePath, sourceEnum}<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.marriott.mmf.{DLTErrorMessage, Error}<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{InputType, JobType, Serializer}<br>import com.marriott.rm.platform.util.ProducerHelperBuilder<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br><br>import scala.io.Source<br><br><br><br>trait GroupBlockFiltrationHelper extends  Serializable {<br><br>  val Json = &quot;json&quot;<br>  val inventories_ITC=&quot;inventories_.inventoryTypeCode&quot;<br>  val RATES_CODE=&quot;rates_.code&quot;<br>  val baseOccupancy=&quot;rates_.guestsByType.count&quot;<br>  @transient val log: CustomLogger<br>  implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;runid&quot;,&quot;propid&quot;,&quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>    &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br>  val kafkaTopicMap: Option[Map[String, String]]<br><br><br>  def groupBlockFiltration(kafkaData: DataFrame)<br>                          (implicit spark: SparkSession,<br>                           connectionConfigDetails: ConnectionConfigDetails,<br>                           jobConfigurationDetails: JobConfigurationDetails,<br>                           logObj:CustomLoggerInfo): Dataset[GroupBlockCollationEvent] = {<br><br><br>    import spark.implicits._<br><br><br>    val groupBlockDataset = kafkaData.map(x =&gt; convertToEntityGroupBlockCollation(x.getString(0)))<br><br>    val groupBlockDatasetValid = groupBlockDataset.filter(x =&gt; x._1.isDefined)<br><br>    val groupBlockDatasetInvalid = groupBlockDataset.filter(x =&gt; x._1.isEmpty).map(x=&gt;x._2).toDF(&quot;value&quot;) // Filter the non processed events and get the payload to push to irrecoverable topic<br>    handleMalFormedEvents(groupBlockDatasetInvalid)<br><br>    val groupBlockDF = groupBlockDatasetValid.toDF(&quot;value&quot;, &quot;payload&quot;)// Don&#39;t drop the payload string for valid event it is used in recoverable event scenario.<br>    log.info(&quot;DataFrame to Entity Conversion&quot;, logObj)<br>    val column: Column = col(&quot;value.hotel&quot;)<br>      .getItem(&quot;propertyCode&quot;)<br>      .getItem(&quot;code&quot;)<br>      .alias(&quot;property-code&quot;)<br><br>    val (mgpPropertyDf, nonMgpPropertyDf) = FilterPropertyCodeUtil.filterPropertyCode(groupBlockDF, column)<br>    val mgpCRSPropDF = mgpPropertyDf<br>      .filter(<br>        col(&quot;value.sourceCRSEnum&quot;).isNotNull &amp;&amp;<br>        col(&quot;value.sourceCRSEnum&quot;) === lit(sourceEnum))<br>    val transactionformedDataDF = transformtoDF(mgpCRSPropDF)<br><br>    val groupBlockCollationDS = transactionformedDataDF.map(row =&gt; convertToFinalDS(row))<br><br>    groupBlockCollationDS<br>  }<br><br>  /**<br>   * Handle MalFormed/ Non parseable(Missing required Attributes) events and push the events to the irrecoverable topic<br>   * @param events - Dataframe of Malformed events under the &#39;value&#39; column<br>   */<br>  def handleMalFormedEvents(events: DataFrame)(implicit<br>                                               sparkSession: SparkSession,<br>                                               jobConfigDetails: JobConfigurationDetails,<br>                                               connectionConfigDetails: ConnectionConfigDetails,<br>  ): Unit ={<br>    val checkpointPath = S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(Default),<br>      jobConfigDetails.jobname,<br>      jobConfigDetails.checkpointdirectory.get(GroupBlockCollationIrrecoverableEntityCheckpointDir))<br>    val schemaPath =<br>      jobConfigDetails.kafkatopicmap.get(IrrecoverableFilePath)<br>    val topic = KafkaProperties.irrecoverableErrorTopic<br>    val irrecoverbleDF = transformNonConvertedEntitiesToIrrecoverableDLTMessage(events,&quot;value&quot;)<br>    val irrecoverableProducer = new ProducerHelperBuilder(irrecoverbleDF, topic, jobTypes = JobType.Streaming)<br>      .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>      .withInputType(InputType.Avro).withIsSingleStreaming(false)<br>      .withCaseClass(Option(classOf[DLTErrorMessage]))<br>      .withFilePath(Option(schemaPath)).withCheckPointPath(Option(checkpointPath))<br>      .build()<br>    irrecoverableProducer.produce()<br>  }<br><br>  /**<br>   * Function to transform the Events to the payload for the Irrecoverable Topic<br>   * @param nonConvertedEntities - DataFrame of the events which were non-parseable<br>   * @param columnWithSourceString - Dataframe col with the events payload<br>   * @param sparkSession - SparkSession<br>   * @param jobConfigDetails - jobConfigurationDetails<br>   * @return - Dataframe for pushing the message to the Irrecoverable Topic<br>   */<br>  def transformNonConvertedEntitiesToIrrecoverableDLTMessage(<br>                                                              nonConvertedEntities: DataFrame,<br>                                                              columnWithSourceString: String<br>                                                            )(implicit<br>                                                              sparkSession: SparkSession<br>                                                            ): DataFrame = {<br>    log.info(&quot;Transform non converted EEH Data to Base DLTErrorMessage Entity &quot;,logObj)<br>    nonConvertedEntities<br>      .select(col(columnWithSourceString))<br>      .withColumn(&quot;errorCode&quot;, lit(&quot;301&quot;))<br>      .withColumn(&quot;exception&quot;, lit(&quot;MalformedEvent&quot;))<br>      .withColumn(<br>        &quot;errorMessage&quot;,<br>        lit(&quot;Event does not have require mandatory parameters&quot;)<br>      )<br>      .withColumnRenamed(<br>        columnWithSourceString,<br>        &quot;mainTopicMessage&quot;<br>      )<br>      .as[Error](Encoders.product[Error])<br>      .map(entity =&gt; DLTErrorMessage(Option(entity)))(<br>        Encoders.product[DLTErrorMessage]<br>      )<br>      .withColumn(&quot;value&quot;, struct(&quot;error&quot;))<br>      .drop(&quot;error&quot;)<br>      .toDF()<br>  }<br><br><br><br><br>  def convertToFinalDS(row: Row): GroupBlockCollationEvent = {<br><br>    log.info(&quot;convertToFinalDS :: Converting dataframe to columnmap - CUD-PAYLOAD to GroupBlockCollationEvent Dataset&quot;, logObj)<br><br><br>    val columns = Seq(&quot;payload&quot;,&quot;actionEnum&quot;, &quot;generationTime&quot;, &quot;confirmationNumber&quot;, &quot;opportunityId&quot;, &quot;quoteId&quot;, &quot;propertyCode&quot;, &quot;groupCode&quot;, &quot;groupName&quot;, &quot;groupType&quot;, &quot;groupEffectiveDate&quot;, &quot;groupExpireDate&quot;, &quot;ratePlanEffectiveDate&quot;, &quot;ratePlanExpireDate&quot;, &quot;inventoryTypeCode&quot;, &quot;ratePlanCode&quot;, &quot;inventoryEffectiveDate&quot;, &quot;inventoryExpireDate&quot;, &quot;verb&quot;, &quot;amount&quot;, &quot;currencyCode&quot;, &quot;dec_pre&quot;, &quot;blocked&quot;, &quot;HotelId&quot;)<br>    val coloumnmap: Map[String, String] = columns.map { fieldName =&gt;<br>      val columnName = fieldName<br>      val value = Option(row.getAs[Any](columnName)).map(_.toString).getOrElse(&quot;null&quot;)<br>      columnName -&gt; value<br>    }.toMap<br><br>    val cUDBaseTable: CUDBaseTable = CUDBaseTable(&quot;&quot;, coloumnmap(&quot;HotelId&quot;), &quot;&quot;, coloumnmap(&quot;generationTime&quot;), coloumnmap(&quot;verb&quot;))<br>    val cudentity: CUDPayload = CUDPayload(cUDBaseTable, coloumnmap, &quot;&quot;, &quot;&quot;)<br>    val GBpayload: GBCollationPayload = GBCollationPayload(coloumnmap(&quot;payload&quot;), &quot;&quot;, null, cudentity)<br><br>    GroupBlockCollationEvent(groupBlockInfo = null, groupBlockPayload = GBpayload)<br>  }<br><br><br>  def convertToEntityGroupBlockCollation(entity: String)<br>                                        (implicit sparkSession: SparkSession): (Option[GroupBlockCollation], String) = {<br>    log.info(&quot;Starting the process of parsing the source data as per the entities mentioned in the [GroupBlock] - convertToEntityGroupBlockCollation()&quot;, logObj)<br>    try {<br>      configureJsonMapper()<br>      val data = Some(JsonUtil.fromJson[GroupBlockCollation](entity))<br><br>      (data, entity)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;KafkaStreamingJob ::: getEventFromString() :: Exception :&quot; + exception, logObj)<br>        (None, entity)<br>    }<br>  }<br>  private def configureJsonMapper(): Unit = {<br>    JsonUtil.mapper.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES,true)<br>    JsonUtil.mapper.configure(DeserializationFeature.FAIL_ON_NULL_CREATOR_PROPERTIES,true)<br>  }<br><br>  val combineIds = udf((ids: Seq[Row]) =&gt; {<br>    if(ids!=null) {<br>      ids.map(id =&gt; (id.getAs[String](&quot;type&quot;), id.getAs[String](&quot;id&quot;))).toMap<br>    }else{<br>      Map.empty[String,String]<br>    }<br>    })<br><br><br>  def transformtoDF(mgpData: DataFrame) = {<br><br>    val explodedDataDF = mgpData.<br>      withColumn(&quot;actionEnum&quot;, col(&quot;value.actionEnum&quot;))<br>      .withColumn(&quot;generationTime&quot;, col(&quot;value.generationTime&quot;))<br>      .withColumn(&quot;ids&quot;, combineIds(col(&quot;value.groupContract.externalIds&quot;)))<br>      .withColumn(&quot;opportunityId&quot;, col(&quot;ids&quot;).getItem(&quot;Opportunity ID&quot;))<br>      .withColumn(&quot;quoteId&quot;, when(col(&quot;ids&quot;).getItem(&quot;Quote ID&quot;).isNotNull, col(&quot;ids&quot;).getItem(&quot;Quote ID&quot;)).otherwise(&quot;&quot;))<br>      .withColumn(&quot;confirmationNumber&quot;, when(col(&quot;ids&quot;).getItem(&quot;Sequence ID&quot;).isNotNull, col(&quot;ids&quot;).getItem(&quot;Sequence ID&quot;)).otherwise(col(&quot;value.groupContract.confirmationNumber.value&quot;)))<br>      .withColumn(&quot;propertyCode&quot;, col(&quot;value.hotel.propertyCode.code&quot;))<br>      .withColumn(&quot;groupCode&quot;, col(&quot;value.groupContract.code&quot;))<br>      .withColumn(&quot;groupName_&quot;, explode(col(&quot;value.groupContract.name.text&quot;)))<br>      .withColumn(&quot;groupName&quot;, explode(col(&quot;groupName_&quot;)))<br>      .withColumn(&quot;groupType&quot;, when(col(&quot;value.groupContract.corporateMarketSegment&quot;)===&quot;Contract&quot;, lit(&quot;Contract&quot;)).otherwise(lit(&quot;Convention&quot;)))<br>      .withColumn(&quot;groupEffectiveDate&quot;, col(&quot;value.groupContract.timespan.start&quot;).cast(&quot;String&quot;))<br>      .withColumn(&quot;groupExpireDate&quot;, col(&quot;value.groupContract.timespan.end&quot;).cast(&quot;String&quot;))<br>      .withColumn(&quot;pricePlan&quot;, explode(col(&quot;value.groupContract.pricePlans&quot;)))<br>      .withColumn(&quot;pricingMethod&quot;, col(&quot;pricePlan.pricing.methodsEnum&quot;))<br>      .withColumn(&quot;rates___&quot;, explode(col(&quot;value.groupContract.pricePlans.priceGridDetails.amountRules&quot;)))<br>      .withColumn(&quot;rates__&quot;, explode(col(&quot;rates___&quot;)))<br>      .withColumn(&quot;rates_&quot;, explode(col(&quot;rates__.inventoryTypes&quot;)))<br>      .withColumn(&quot;ratePlanEffectiveDate&quot;, explode(col(&quot;rates___.timespan.start&quot;)))<br>      .withColumn(&quot;ratePlanExpireDate&quot;, explode(col(&quot;rates___.timespan.end&quot;)))<br>      .withColumn(&quot;counters_&quot;, explode(col(&quot;value.offer.productUses&quot;)))<br>      .withColumn(&quot;inventories_&quot;, explode(col(&quot;value.products&quot;)))<br>      .withColumn(&quot;inventoryTypeCode&quot;, col(inventories_ITC))<br>      .withColumn(&quot;ratePlanCode&quot;, col(&quot;value.groupContract.rateReferences.ratePlanCode&quot;))<br>      .withColumn(&quot;inventoryEffectiveDate&quot;, col(&quot;counters_.timespan.start&quot;))<br>      .withColumn(&quot;inventoryExpireDate&quot;, col(&quot;counters_.timespan.end&quot;))<br>      .drop(&quot;value&quot;, &quot;rates___&quot;, &quot;rates__&quot;, &quot;groupName_&quot;, &quot;counters_.timespan&quot;, &quot;ids&quot;, &quot;pricePlan&quot;)<br><br>      log.info(<br>      s&quot;Columns of exploded dataframe :: ${explodedDataDF.columns.mkString(&quot;Array(&quot;, &quot;, &quot;, &quot;)&quot;)}&quot;,<br>      logObj<br>    )<br><br>    val finaldf = explodedDataDF.dropDuplicates()<br>    val absenceDataDF = finaldf.filter(col(RATES_CODE).isNull<br>        &amp;&amp; col(inventories_ITC) == col(&quot;counters_.inventoryTypeCode&quot;))<br>      .withColumn(&quot;verb&quot;, when(col(&quot;actionEnum&quot;) === &quot;Create&quot;, lit(&quot;B&quot;)).when(col(&quot;actionEnum&quot;) === &quot;Update&quot;, lit(&quot;A&quot;)).otherwise(lit(&quot;D&quot;)))<br>      .withColumn(&quot;amount&quot;, lit(null))<br>      .withColumn(&quot;currencyCode&quot;, lit(null))<br>      .withColumn(&quot;dec_pre&quot;, lit(null))<br>      .withColumn(&quot;blocked&quot;, col(&quot;counters_.countersPerInventoryType.deducted&quot;))<br><br><br>    val validGuestsCount = finaldf.withColumn(&quot;guestCount&quot;,<br>      when(col(baseOccupancy).isNull,0)<br>        .otherwise(col(baseOccupancy)))<br>    val validDataDF = validGuestsCount.filter(col(RATES_CODE).isNotNull)<br>      .filter(col(RATES_CODE) === col(inventories_ITC)<br>        &amp;&amp; col(inventories_ITC) === col(&quot;counters_.inventoryTypeCode&quot;))<br>      .withColumn(&quot;verb&quot;, when(col(&quot;actionEnum&quot;) === &quot;Create&quot;, lit(&quot;C&quot;)).when(col(&quot;actionEnum&quot;) === &quot;Update&quot;, lit(&quot;U&quot;)).otherwise(lit(&quot;D&quot;)))<br>      .withColumn(&quot;amount&quot;,<br>        when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null))<br>          .when(col(&quot;pricingMethod&quot;) === &quot;PerProductByOccupancy&quot; &amp;&amp; col(&quot;guestCount&quot;) === 1, col(&quot;rates_.amountValue.value&quot;))<br>          .when(col(&quot;pricingMethod&quot;) === &quot;PerProduct&quot;, col(&quot;rates_.amountValue.value&quot;))<br>      )<br>      .withColumn(&quot;currencyCode&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;rates_.amountValue.currencyCode&quot;)))<br>      .withColumn(&quot;dec_pre&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;rates_.amountValue.numberOfDecimals&quot;)))<br>      .withColumn(&quot;blocked&quot;, when(col(&quot;actionEnum&quot;) === &quot;Delete&quot;, lit(null)).otherwise(col(&quot;counters_.countersPerInventoryType.deducted&quot;)))<br>      .drop(col(&quot;guestCount&quot;))<br>    log.info(&quot;Union of both Absense and Valid Dataframes&quot;, logObj)<br>    var dataDF: Dataset[Row] = absenceDataDF.union(validDataDF)<br>    dataDF = dataDF<br>      .withColumn(&quot;HotelId&quot;, concat(col(&quot;propertyCode&quot;), col(&quot;groupCode&quot;), col(&quot;confirmationNumber&quot;)))<br>      .drop(&quot;rates_&quot;, &quot;inventories_&quot;, &quot;counters_&quot;, &quot;pricingMethod&quot;)<br><br>    log.info(<br>      s&quot;Columns of dataDf dataframe before Column map :: ${dataDF.columns.mkString(&quot;Array(&quot;, &quot;, &quot;, &quot;)&quot;)}&quot;,<br>      logObj)<br>    dataDF<br><br><br><br><br>  }<br>}<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>groupblockhelper<br><br><br>package com.marriott.mgpapp.spark.groupblock.collation.helper<br>import java.sql.{Connection, Statement}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants._<br>import java.sql.{Date, ResultSet, Timestamp}<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails, MessageBrokerDetails}<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br>import com.marriott.elevate.spark.framework.utils.DateUtil._<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbReader<br>import com.marriott.mgpapp.spark.groupblock.collation.udfs.SparkUdfs._<br>import com.marriott.mgpapp.spark.groupblock.collation.dao.GroupBlockRow<br>import com.marriott.mgpapp.spark.groupblock.Constants._<br>import com.marriott.utility.CustomLogger<br>import org.joda.time.DateTime<br>import org.joda.time.format.ISODateTimeFormat<br>import scala.util.{Failure, Success, Try}<br><br><br>trait GroupBlockHelper extends LazyLogging with Serializable {<br><br>  val schemaMap: Map[String, String]<br>  val log: CustomLogger<br>  val jobName:String<br>  val jobIdentifier:String<br>  val srf=&quot;Succesfully Retreived from :::&quot;<br>  val fip=&quot;Failed to initialize pool: Connection to POSTGRES refused.PSQL Exception &quot;<br><br>  def findLatestRecords(groupedKeyValue: (String, String, String, String, Date, String), records: Iterator[GroupBlockCollationSchema]): GroupBlockCollationSchema = {<br>    val latestRecord = records.reduceLeft((x, y) =&gt; getLatestRecord(x, y))<br>    latestRecord<br><br>  }<br><br>  def getLatestRecord(x: GroupBlockCollationSchema, y: GroupBlockCollationSchema): GroupBlockCollationSchema = {<br>    val latestEvent = if (getLoadTimeStamp(x).isAfter(getLoadTimeStamp(y))) x else y<br>    latestEvent<br>  }<br><br>  def getLoadTimeStamp(event: GroupBlockCollationSchema): DateTime = {<br>    getTS(event.transmissionTimestamp).get<br>  }<br><br>  def getTS(timestamp: String): Option[DateTime] = timestamp match {<br>    case &quot;&quot; =&gt; None<br>    case _ =&gt;<br>      Try(ISODateTimeFormat.dateTimeParser.parseDateTime(timestamp)) match {<br>        case Success(t) =&gt; Some(t)<br>        case Failure(_) =&gt; None<br>      }<br>  }<br><br>  def calculateBlockedRevenue(gbDataset: Dataset[GroupBlockCollationEvent])<br>                             (implicit jobConfigurationDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails,<br>                              spark: SparkSession,<br>                              logObj:CustomLoggerInfo):Dataset[(GroupBlockRow, String, String, Boolean)] = {<br>    import spark.implicits._<br>    log.info(&quot;Calculating blocked Revenue for Group Block &quot; + &quot;- calculateBlockedRevenue()&quot;, logObj)<br>    log.info(&quot;Starting here :: creating key for transform&quot;, logObj)<br>    val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br>   val reqPropGBDataWithSrcIndDF: Dataset[GroupBlockEntityNew] = getReqPropGBDataWithSrcIndDF(dataGB, schemaMap)<br>    val invDtLevelDataDS = reqPropGBDataWithSrcIndDF.flatMap(x =&gt; {<br>      actDF(x, spark)<br>    })<br><br>    val invDtLevel = invDtLevelDataDS.as[GroupBlockEntityNewUpdated]<br>    val invDtLevelDataWithCurCodeDS = getInvDtLevelDataWithCurCodeDS(invDtLevel, schemaMap)<br>    val finalRevenue = processRevenueData(invDtLevelDataWithCurCodeDS, jobName, jobIdentifier)<br><br>    finalRevenue<br>  }<br><br>  def transformGroupBlockRecords(key: String, iterator: Iterator[GroupBlockCollationEvent])<br>                                (implicit connectionConfigDetails: ConnectionConfigDetails,<br>                                 sparkSession: SparkSession,jobConfigurationDetails: JobConfigurationDetails,logObj:CustomLoggerInfo)<br>  : List[GroupBlockEntity] = {<br><br>    import sparkSession.implicits._<br>    log.info(&quot; Transforming groupBlock Records at :: transformGroupBlockRecords()&quot;, logObj)<br><br>    val keyValuePair = getDbLookupPjGb(key, new DbReader(schemaMap))<br>    if(keyValuePair.isDefined) {<br>      val iteratorSet = iterator.duplicate<br><br>      val gbList = mapGroupBlockList(iteratorSet._1, keyValuePair.get)<br>      val PhMSTsList = filterPhMSTsList(gbList)<br>      val gbEntity = getGbEntity(PhMSTsList)<br>      val entity = processGbEntity(Map(&quot;keyValueSchema&quot; -&gt; keyValuePair.get,&quot;schemaMap&quot; -&gt;schemaMap), gbEntity, key, iteratorSet._2)<br>      entity<br>    }else{<br>      iterator.toList.map(grpBlockEvent =&gt; GroupBlockEntity(payload = grpBlockEvent.groupBlockPayload.payLoad,isRecoverable = true)).distinct<br>    }<br>  }<br><br>  def mergeToSingleRecord(row: GroupBlockData, records: Iterator[Row]): (GroupBlockRow, String, String,Boolean) = {<br>    val jsonDataString = records.map(row =&gt; row.getAs(GroupBlockEventJson).toString).mkString(&quot;|&quot;)<br>    val groupBlockRow = GroupBlockRow(row.pgbPropCode, row.pgbGroupCode, Some(row.pgbCnfrmNum),row.pgbGrpCtrctInd,<br>      row.startDateTimePGB, row.pgbEndDt, row.pgbOpptyId, row.pgbQuoteId, row.pgbGroupName, row.tsPgbTransmission,<br>      row.actionPgb, row.lastUpByPGB, new Timestamp(new java.util.Date().getTime))<br>    val srcEventPayload = row.sourcePayload<br>    (groupBlockRow, jsonDataString, srcEventPayload,false)<br>  }<br><br><br>  def getDbLookupPjGb(key: String, dbReader:DbReader<br>                     )(implicit connConfig: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo): Option[Map[String, String]] = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getDbLookupPjGb() on table - &quot; + schemaMap.get(TGTGroupBlockTable).get, logObj)<br>      val propCode = key.substring(0, 5)<br>      val groupCode = key.substring(5, 15)<br>      val confirmationNumber = key.substring(15, key.length)<br>      val query = &quot;SELECT * FROM &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(TGTGroupBlockTable).get) + &quot; where pgb_prop_code=&quot; + &quot;&#39;&quot; + propCode + &quot;&#39;&quot; + &quot; and pgb_group_code=&quot; + &quot;&#39;&quot; + groupCode + &quot;&#39;&quot; + &quot; and pgb_cnfrm_num=&quot; + &quot;&#39;&quot; +  confirmationNumber + &quot;&#39;&quot;<br>      log.info(&quot;Query in getDbLookupPjGb() ===&gt;&quot; + query, logObj)<br>    dbReader.open()<br>    val LookupPjGb =   dbReader.processLookupPjGb(query)<br>    LookupPjGb<br>  }<br><br>  def getDbLookupBrandCode(key: String,dbReader:DbReader<br>                          )(implicit connConfig: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo): Option[Boolean] = {<br>    log.info(&quot;Performing DB Lookup at ::: getDbLookupBrandCode() on table - &quot; + schemaMap.get(PropertyTableIN).get, logObj)<br>    val propCode = key.substring(0, 5)<br>    val query = &quot;select pt_prop_code, pt_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(PropertyTableIN).get) + &quot; where pt_prop_code=&quot; + &quot;&#39;&quot; + propCode + &quot;&#39;&quot; + &quot; and pt_brand_code in (select uc_brand_code from &quot; + connConfig.schemaconfig.get(&quot;SchemaName&quot;).concat(&quot;.&quot;).concat(schemaMap.get(UserControlTableIN).get) + &quot; where uc_brand_oy_cmnty=&#39;Y&#39; and uc_prop_code=&#39;ALL00&#39;)&quot;<br>    log.info(&quot;Query in getDbLookupBrandCode() ===&gt;&quot; + query, logObj)<br>    dbReader.open()<br>    val answer =   dbReader.processBrandCode(query)<br>    answer<br><br>  }<br><br>  def getLookupDataPropParamAssignMap(propCode: String,dbReader:DbReader)<br>                                     (implicit connConfig: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo): Option[Map[String, String]] = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getLookupDataPropParamAssignMap() on table - prop_param_assign&quot;, logObj)<br><br>    val sqlQuery = s&quot;select $PpaPropCode  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTableIN).get} where $PpaParamId=&#39;DATASRCIND&#39; and $PpaParamValue=&#39;S&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;<br>    log.info(&quot;Query in getLookupDataPropParamAssignMap() ===&gt;&quot; + sqlQuery, logObj)<br>    dbReader.open()<br>      val PropParamAssign = dbReader.processPropParamAssignMap(sqlQuery)<br>    PropParamAssign<br>  }<br><br>  def getLookupDataPropParamAssignWithPropcodeAndParamValue(propCode: String, dbReader:DbReader)(implicit connConfig: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo): Option[String] = {<br><br>    log.info(&quot;Performing DB Lookup at ::: getLookupDataPropParamAssignWithPropcodeAndParamValue() on table - prop_param_assign&quot;, logObj)<br>      val sqlQuery = s&quot;select $PpaParamValue  from ${schemaMap(SchemaName)}.${schemaMap.get(PropParamAssignTableIN).get} where $PpaParamId=&#39;SELLCURCD&#39; and $PpaActiveInd=&#39;Y&#39; and  $PpaPropCode = &#39;$propCode&#39; &quot;<br>      log.info(&quot;Query in getLookupDataPropParamAssignWithPropcodeAndParamValue() ===&gt;&quot; + sqlQuery, logObj)<br>     dbReader.open()<br>      val answer =   dbReader.processPropParam(sqlQuery)<br>      answer<br><br>  }<br><br>  def summarizeDetail(groupByKey: (String, String, String, Date, Date,<br>    String, String, String, String, String, String, String, String, Date, String, Option[Int]),<br>                      rec: Iterator[GroupBlockEntityUpdatedCurrencyCode]): GroupBlockCollationSchema = {<br><br>    val records = rec.toSeq<br><br>    val recordsWithNonZeroAmt = records.filter(x =&gt; x.Amt &gt; 0 &amp;&amp; (x.inventoryDate.after(x.ratePlanEffectiveDate) || x.inventoryDate.equals(x.ratePlanEffectiveDate)) &amp;&amp; x.inventoryDate.before(x.ratePlanExpireDate))<br>    val filteredRecords = if (recordsWithNonZeroAmt.isEmpty) {<br>      records<br>    }<br>    else {<br>      recordsWithNonZeroAmt<br>    }<br>    val totalAmt = filteredRecords.map(f =&gt; f.Amt).sum<br>    val recordCount = filteredRecords.size<br>    val avg = totalAmt / recordCount<br>    val blockedRevenue = avg * filteredRecords.head.blockedRooms.getOrElse(0)<br>    val record = filteredRecords.head<br>    val recordTimestamp = record.transmissionTimestamp.replace(&quot;T&quot;, &quot; &quot;).replace(&quot;Z&quot;, &quot;&quot;)<br>    GroupBlockCollationSchema(record.groupCode, record.propertyCode, record.confirmationNumber, record.groupEffectiveDate, record.groupExpireDate,<br>      record.groupIndicator, record.groupSourceIndicator, record.opportunityId, record.quoteId, record.groupEventName, record.transmissionTimestamp,<br>      record.currencyCode, record.actionIndicator, record.inventoryDate, record.inventoryTypeCode, record.blockedRooms.getOrElse(0), avg, recordCount, recordTimestamp, blockedRevenue,record.payload)<br>  }<br><br><br><br>  /**<br>   * Parse the default event info intoJson format<br>   *<br>   * @param entity<br>   */<br><br>  def actDF(actualDF:GroupBlockEntityNew,spark:SparkSession):Seq[GroupBlockEntityNewUpdated]= {<br>    import spark.implicits._<br>    val groupBlockList = if(actualDF.isRecoverable){<br>      Seq(GroupBlockEntityNewUpdated(payload = actualDF.payload, isRecoverable = actualDF.isRecoverable))<br>    } else if (actualDF.InvDateSpan &gt; 0) {<br>      for {i &lt;- 1 to actualDF.InvDateSpan} yield {<br>        val GBEntity = GroupBlockEntityNewUpdated(<br>          actualDF.actionIndicator,<br>          actualDF.transmissionTimestamp,<br>          actualDF.opportunityId,<br>          actualDF.quoteId,<br>          actualDF.confirmationNumber,<br>          actualDF.propertyCode,<br>          actualDF.groupCode,<br>          actualDF.groupEffectiveDate,<br>          actualDF.groupExpireDate,<br>          actualDF.groupEventName,<br>          actualDF.groupEventType,<br>          actualDF.inventoryTypeCode,<br>          actualDF.ratePlanCode,<br>          actualDF.ratePlanEffectiveDate,<br>          actualDF.ratePlanExpireDate,<br>          actualDF.amount,<br>          actualDF.currencyCode,<br>          actualDF.decimalPrecision,<br>          actualDF.blockedRooms,<br>          actualDF.inventoryEffectiveDate,<br>          actualDF.inventoryExpireDate,<br>          actualDF.ppa_prop_code,<br>          actualDF.data_source_ind,<br>          actualDF.groupIndicator,<br>          actualDF.groupSourceIndicator,<br>          actualDF.inventoryDate,<br>          actualDF.Amt,<br>          actualDF.payload<br>        )<br>        GBEntity<br>      }<br>    } else {<br>      Seq(GroupBlockEntityNewUpdated(<br>        actualDF.actionIndicator,<br>        actualDF.transmissionTimestamp,<br>        actualDF.opportunityId,<br>        actualDF.quoteId,<br>        actualDF.confirmationNumber,<br>        actualDF.propertyCode,<br>        actualDF.groupCode,<br>        actualDF.groupEffectiveDate,<br>        actualDF.groupExpireDate,<br>        actualDF.groupEventName,<br>        actualDF.groupEventType,<br>        actualDF.inventoryTypeCode,<br>        actualDF.ratePlanCode,<br>        actualDF.ratePlanEffectiveDate,<br>        actualDF.ratePlanExpireDate,<br>        actualDF.amount,<br>        actualDF.currencyCode,<br>        actualDF.decimalPrecision,<br>        actualDF.blockedRooms,<br>        actualDF.inventoryEffectiveDate,<br>        actualDF.inventoryExpireDate,<br>        actualDF.ppa_prop_code,<br>        actualDF.data_source_ind,<br>        actualDF.groupIndicator,<br>        actualDF.groupSourceIndicator,<br>        actualDF.inventoryDate,<br>        actualDF.Amt,<br>        actualDF.payload<br>      ))<br>    }<br>    groupBlockList.distinct<br>  }<br>  def checkForNull(str: Any): String = if (str == null) &quot;&quot; else str.toString<br><br>  def mapGroupBlockList(iterator: Iterator[GroupBlockCollationEvent], keyValuePair: Map[String, String]): List[GroupBlockCollationEvent] = {<br>    iterator.toList.map { event =&gt;<br>      val gbCollEvent = event<br>      val cudpLoad = gbCollEvent.getPayload.groupBlockCUD.copy(columnMap = gbCollEvent.getPayload.groupBlockCUD.columnMap ++ keyValuePair)<br>      val gCollationPayLoad = gbCollEvent.groupBlockPayload.copy(groupBlockCUD = cudpLoad)<br>      gbCollEvent.copy(groupBlockPayload = gCollationPayLoad)<br>    }<br>  }<br><br>  def filterPhMSTsList(gbList: List[GroupBlockCollationEvent]): List[GroupBlockCollationEvent] = {<br>    gbList.filter { x =&gt;<br>      val loadTs = x.getPayload.groupBlockCUD.baseTable.loadTimeStamp<br>      val phDbTs = x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;pgb_transmission_ts&quot;, NULL)<br>      phDbTs == NULL || getTimestamp(loadTs, DateFormat).get.after(getTimestamp(phDbTs,UtcTimestampFormat).get)<br><br><br>    }<br>  }<br><br>  def getGbEntity(PhMSTsList: List[GroupBlockCollationEvent])(implicit logObj:CustomLoggerInfo): List[GroupBlockEntity] = {<br>    PhMSTsList.flatMap(x =&gt; {<br>      val gBEvents = if (x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == Delete) {<br>        Set(GroupBlockEntity(<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          None, None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          None, None,<br>          None, None, None, None, None, None, None, None, None, None, None,x.groupBlockPayload.payLoad))<br>      }<br>      else if ((x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == UdateRateAbsence) || (x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;) == CreateRateAbsence)) {<br>        log.info(&quot; Rate Absence Records:: InventoryTypeCode = &quot; + x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;), logObj)<br>        Set(GroupBlockEntity(Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;opportunityId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;quoteId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupName&quot;, &quot;&quot;).toString()),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupType&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;)),<br>          None, None, None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;amount&quot;, &quot;&quot;).toInt),<br>          None,<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;dec_pre&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;blocked&quot;, &quot;&quot;).toInt),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryExpireDate&quot;, &quot;&quot;), DATE_FORMAT),x.groupBlockPayload.payLoad))<br>      }<br>      else {<br>        Set(GroupBlockEntity(Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;verb&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;generationTime&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;opportunityId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;quoteId&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;confirmationNumber&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;propertyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupName&quot;, &quot;&quot;).toString()),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;groupType&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryTypeCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanCode&quot;, &quot;&quot;)),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;ratePlanExpireDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;amount&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;currencyCode&quot;, &quot;&quot;)),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;dec_pre&quot;, &quot;&quot;).toInt),<br>          Some(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;blocked&quot;, &quot;&quot;).toInt),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryEffectiveDate&quot;, &quot;&quot;), DATE_FORMAT),<br>          getDate(x.getPayload.groupBlockCUD.columnMap.getOrElse(&quot;inventoryExpireDate&quot;, &quot;&quot;), DATE_FORMAT),x.groupBlockPayload.payLoad))<br>      }<br><br>      /* Added below logic to include &#39;CMP&#39; groupBlock which has rooms but no revenue for the InventoryTypeCode.<br>              Updating actionIndicator from A to U*/<br>      val groupBlockUpdatedEvents = gBEvents.toList<br>        .groupBy(x =&gt; (x.groupCode, x.propertyCode)).flatMap { case (key, events) =&gt; {<br><br>          if (key._1.get.equalsIgnoreCase(&quot;CMP&quot;)) {<br>            val UpdateRateAbsence = events.filter(_.actionIndicator.get.equals(&quot;A&quot;))<br>            val UpdateEvents = events.filter(_.actionIndicator.get.equals(&quot;U&quot;))<br><br>            if (UpdateEvents.isEmpty &amp;&amp; UpdateRateAbsence.nonEmpty) {<br>              List(UpdateRateAbsence.head.copy(actionIndicator = Some(&quot;U&quot;)))<br>            } else events<br>          } else events<br>        }<br>        }<br><br>        logger.info(s&quot;groupBlockUpdatedEvents :: $groupBlockUpdatedEvents&quot;)<br><br><br>      groupBlockUpdatedEvents<br><br>    }).filter(_.actionIndicator != UdateRateAbsence).filter(_.actionIndicator != CreateRateAbsence)<br>  }<br><br><br>  def processGbEntity(keyValueSchema: Map[String, Map[String, String]], gbEntity: List[GroupBlockEntity], key: String, iterator: Iterator[GroupBlockCollationEvent])<br>                     (implicit connectionConfigDetails: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo): List[GroupBlockEntity] = {<br><br>    val keyValuePair = keyValueSchema(&quot;keyValueSchema&quot;)<br>    val schemaMap = keyValueSchema(&quot;schemaMap&quot;)<br>    val lookupSuccess = getDbLookupBrandCode(key, new DbReader(schemaMap))<br><br>    if (lookupSuccess.isEmpty) {<br>      return iterator.toList.map(x =&gt; GroupBlockEntity(payload = x.groupBlockPayload.payLoad, isRecoverable = true))<br>    }<br><br>    if (!lookupSuccess.get) {<br>      return List()<br>    }<br><br>    val shouldProcess = keyValuePair(&quot;pgb_transmission_ts&quot;) != NULL &amp;&amp; gbEntity.nonEmpty &amp;&amp;<br>      !gbEntity.head.actionIndicator.contains(Delete) &amp;&amp;<br>      !gbEntity.head.actionIndicator.contains(UdateRateAbsence) &amp;&amp;<br>      !gbEntity.head.actionIndicator.contains(CreateRateAbsence)<br><br>    val entities = if (shouldProcess) {<br>      val requiredFields = gbEntity.filterNot(x =&gt; x.actionIndicator.contains(Delete)).head<br>      val (leftStartDate, leftEndDate) = getLeftDates(keyValuePair, requiredFields)<br>      val (rightStartDate, rightEndDate) = getRightDates(keyValuePair, requiredFields)<br>      val initialDeleteSeq = createDeleteSeq(requiredFields, leftStartDate, leftEndDate)<br>      val laterDeleteSeq = createDeleteSeq(requiredFields, rightStartDate, rightEndDate)<br>      initialDeleteSeq ++ laterDeleteSeq ++ gbEntity<br>    } else {<br>      gbEntity<br>    }<br><br>    val DeletedEvents = entities.toSet.toList<br>    if (DeletedEvents.size &gt; 0) DeletedEvents.toSet.toList else entities.toSet.toList<br>  }<br><br>  def getLeftDates(keyValuePair: Map[String, String], requiredFields: GroupBlockEntity): (Date, Date) = {<br>    if (Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)).before(requiredFields.groupEffectiveDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))<br>    }<br>    else if (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)).before(requiredFields.groupEffectiveDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), requiredFields.groupEffectiveDate.get)}<br>    else {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)))}}<br><br>  def getRightDates(keyValuePair: Map[String, String], requiredFields: GroupBlockEntity): (Date, Date) = {<br>    if (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)).after(requiredFields.groupExpireDate.get)) {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))<br>    }<br>    else if (requiredFields.groupExpireDate.get.before(Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))) {<br>      (requiredFields.groupExpireDate.get, Date.valueOf(keyValuePair(&quot;pgb_end_dt&quot;)))}<br>    else {<br>      (Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)), Date.valueOf(keyValuePair(&quot;pgb_start_dt&quot;)))}}<br><br>  def createDeleteSeq(requiredFields: GroupBlockEntity, startDate: Date, endDate: Date): List[GroupBlockEntity] = {<br>    List(GroupBlockEntity(Some(&quot;DS&quot;), requiredFields.transmissionTimestamp, None, None,<br>      requiredFields.confirmationNumber, requiredFields.propertyCode, requiredFields.groupCode, Some(startDate),<br>      Some(endDate), None, Some(&quot;CONVENTION&quot;), Some(&quot;XXX&quot;), None, Some(startDate), Some(endDate), Some(1),<br>      Some(&quot;XXX&quot;), None, Some(1), Some(startDate), Some(endDate)) )<br>      .filterNot(x =&gt; x.groupEffectiveDate == x.groupExpireDate)<br>  }<br><br>  def getReqPropGBDataWithSrcIndDF(dataGB: Dataset[GroupBlockEntity], schemaMap: Map[String, String])(implicit spark: SparkSession,jobConfigurationDetails: JobConfigurationDetails,<br>                                                                                                      connectionConfigDetails: ConnectionConfigDetails,logObj:CustomLoggerInfo): Dataset[GroupBlockEntityNew] = {<br>    import spark.implicits._<br><br>    val recoverableDataGB: Dataset[GroupBlockEntityNew] = dataGB.filter(d =&gt; d.isRecoverable).map(x =&gt; GroupBlockEntityNew(x.payload, true))<br>    log.info(&quot;getReqPropGBDataWithSrcIndDF&quot;,logObj)<br>    val lookupDataPropParamAssign: Dataset[GroupBlockEntityPropCode] = dataGB.filter(d =&gt; d.isRecoverable == false).groupByKey(x =&gt; (x.propertyCode.get))<br>      .mapGroups { case (rowid, event) =&gt; event.toList }.flatMap(record =&gt; {<br>      val ppaPropCode = record.head.propertyCode.get<br>      val reqPropGBData = getLookupDataPropParamAssignMap(ppaPropCode,  new DbReader(schemaMap))<br>      if (reqPropGBData.isDefined) {<br>        record.map(record =&gt; {<br>          val GBParamAssignDS = GroupBlockEntityPropCode(record.actionIndicator, record.transmissionTimestamp,<br>            record.opportunityId, record.quoteId, record.confirmationNumber, record.propertyCode, record.groupCode,<br>            record.groupEffectiveDate, record.groupExpireDate, record.groupEventName, record.groupEventType,<br>            record.inventoryTypeCode, record.ratePlanCode, record.ratePlanEffectiveDate, record.ratePlanExpireDate,<br>            record.amount, record.currencyCode, record.decimalPrecision, record.blockedRooms,<br>            record.inventoryEffectiveDate, record.inventoryExpireDate,<br>            reqPropGBData.get.getOrElse(PpaPropCode, NULL), record.payload)<br>          GBParamAssignDS<br>        })<br>      } else {<br>        record.map(x =&gt; GroupBlockEntityPropCode(payload = x.payload, isRecoverable = true))<br>      }<br>    })<br><br>    val lookUpDataPropParamAssignRecoverable:Dataset[GroupBlockEntityNew] = lookupDataPropParamAssign.filter(d=&gt;d.isRecoverable).map(x=&gt;GroupBlockEntityNew(x.payload,true))<br>    val reqPropGBDataWithSrcIndDF:Dataset[GroupBlockEntityNew] = lookupDataPropParamAssign.filter(d=&gt;d.isRecoverable==false).toDF().withColumn(DataSourceInd, when(trim(col(<br>        PropertyCodeConst)) === trim(col(PpaPropCode)), lit(SgdsInd))<br>        .otherwise(lit(NonSgdsInd)))<br>      .withColumn(GroupIndicator, when(upper(col(GroupEventType)) === lit(Convention), GroupInd)<br>        .when(upper(col(GroupEventType)) === lit(Contract), ContractInd))<br>      .withColumn(GroupSrcIndicator, when(upper(col(GroupEventType)) === lit(Convention) and<br>        upper(col(DataSourceInd)) === lit(SgdsInd), lit(SgdsInd))<br>        .when(upper(col(GroupEventType)) === lit(Convention) and<br>          upper(col(DataSourceInd)) === lit(NonSgdsInd), lit(NonSgdsInd))<br>        .when(upper(col(GroupEventType)) === lit(Contract), lit(ContractInd)))<br>      .withColumn(InvDateSpan, datediff(when(col(InventoryExpireDate).isNotNull,<br>        col(InventoryExpireDate)).otherwise(current_date),<br>        when(col(InventoryEffectiveDate).isNotNull, col(InventoryEffectiveDate)).otherwise(current_date)))<br>      .withColumn(InventoryDateArray, getInventoryDateList(col(ActionIndicator),<br>        when(col(InventoryEffectiveDate).isNotNull, col(InventoryEffectiveDate)).otherwise(current_date), col(InvDateSpan)))<br>      .withColumn(InvDate, explode(col(InventoryDateArray)))<br>      .withColumn(Amt, getAmount(when(col(Amount).isNotNull, col(Amount)).otherwise(0),<br>        when(col(DecPrec).isNotNull, col(DecPrec)).otherwise(0)))<br>      .withColumn(&quot;AdditionalIndicator&quot;, when((col(ActionIndicator) === &quot;A&quot; &amp;&amp; col(BlockedRooms).eq(Some(0)))<br>        || (col(ActionIndicator) === &quot;B&quot; &amp;&amp; col(BlockedRooms).eq(Some(0))), lit(0)).otherwise(1))<br>      .as[GroupBlockEntityAdj]<br>      .filter(col(&quot;AdditionalIndicator&quot;) === 1)<br>      .drop(&quot;AdditionalIndicator&quot;, InventoryDateArray)<br>      .as[GroupBlockEntityNew]<br><br>    recoverableDataGB.unionByName(lookUpDataPropParamAssignRecoverable).unionByName(reqPropGBDataWithSrcIndDF)<br>  }<br><br>  def getInvDtLevelDataWithCurCodeDS(invDtLevel: Dataset[GroupBlockEntityNewUpdated], schemaMap: Map[String,String])(implicit spark: SparkSession,jobConfig: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails,logObj:CustomLoggerInfo): Dataset[GroupBlockEntityUpdatedCurrencyCode] = {<br>    import spark.implicits._<br>    val invtDtLevelRecoverable = invDtLevel.filter(x=&gt;x.isRecoverable).map(x=&gt;GroupBlockEntityUpdatedCurrencyCode(x.payload,true))<br>    log.info(&quot;getInvDtLevelDataWithCurCodeDS()&quot;,logObj)<br><br>    val invtDtLevelTransformed = invDtLevel.filter(x=&gt;x.isRecoverable==false).groupByKey(x =&gt; (x.propertyCode))<br>      .mapGroups { case (_, event) =&gt; event.toList }.coalesce(5).flatMap(records =&gt; {<br>        val propertyCode = records.head.propertyCode<br>        val currencyCode = getLookupDataPropParamAssignWithPropcodeAndParamValue(propertyCode,  new DbReader(schemaMap))(connectionConfigDetails: ConnectionConfigDetails, jobConfig: JobConfigurationDetails,logObj:CustomLoggerInfo)<br>      if(currencyCode.isDefined) {<br>        log.info(&quot;currencyCode ::&quot; + currencyCode, logObj)<br><br>        records.map(record =&gt; {<br>          log.info(&quot;Record currencyCode ::&quot; + {<br>            record.currencyCode<br>          }, logObj)<br>          val GroupBlockEntityUpdatedCurrencyCodeRecords = GroupBlockEntityUpdatedCurrencyCode(record.actionIndicator,<br>            record.transmissionTimestamp, record.opportunityId, record.quoteId, record.confirmationNumber,<br>            record.propertyCode, record.groupCode, record.groupEffectiveDate,<br>            record.groupExpireDate, record.groupEventName, record.groupEventType, record.inventoryTypeCode,<br>            record.ratePlanCode, record.ratePlanEffectiveDate, record.ratePlanExpireDate,<br>            record.amount, if (record.currencyCode != null) record.currencyCode else currencyCode.get, record.decimalPrecision, record.blockedRooms,<br>            record.inventoryEffectiveDate, record.inventoryExpireDate, record.ppa_prop_code,<br>            record.data_source_ind,  record.groupIndicator, record.groupSourceIndicator, record.inventoryDate, record.Amt, record.payload)<br><br>          log.info(&quot;GroupBlockEntityUpdatedCurrencyCodeRecords :: &quot; + GroupBlockEntityUpdatedCurrencyCodeRecords, logObj)<br>          GroupBlockEntityUpdatedCurrencyCodeRecords<br>        })<br>      }else{<br>        records.map(record=&gt;GroupBlockEntityUpdatedCurrencyCode(payload = record.payload, isRecoverable = true))<br>      }<br>      })<br>    invtDtLevelRecoverable.union(invtDtLevelTransformed).coalesce(5)<br>  }<br><br>  def processRevenueData(invDtLevelDataWithCurCodeDS: Dataset[GroupBlockEntityUpdatedCurrencyCode], jobName: String, jobIdentifier: String)(implicit spark: SparkSession,logObj:CustomLoggerInfo): Dataset[(GroupBlockRow,String,String,Boolean)] = {<br>    import spark.implicits._<br><br>    val invDtLevelDataWithCurCodeDSRecoverable = invDtLevelDataWithCurCodeDS.filter(x=&gt;x.isRecoverable).map(x=&gt;(new GroupBlockRow(&quot;&quot;,&quot;&quot;,None,&quot;&quot;,None,None,&quot;&quot;,&quot;&quot;,&quot;&quot;,None,&quot;&quot;,&quot;&quot;,new Timestamp(System.currentTimeMillis())),&quot;&quot;,x.payload,true))<br><br>    val avgBlockedActualRevenueDS: Dataset[GroupBlockCollationSchema] = invDtLevelDataWithCurCodeDS.filter(x=&gt;x.isRecoverable==false)<br>      .groupByKey(record =&gt; (record.groupCode, record.propertyCode, record.confirmationNumber, record.groupEffectiveDate, record.groupExpireDate,<br>        record.groupIndicator, record.groupSourceIndicator, record.opportunityId, record.quoteId, record.groupEventName, record.transmissionTimestamp,<br>        record.currencyCode, record.actionIndicator, record.inventoryDate, record.inventoryTypeCode, record.blockedRooms))<br>      .mapGroups(summarizeDetail).coalesce(5)<br><br>    val getLatestRecsDS: Dataset[GroupBlockCollationSchema] = avgBlockedActualRevenueDS.as[GroupBlockCollationSchema]<br>      .groupByKey(record =&gt; (record.propertyCode, record.groupCode, record.confirmationNumber, record.quoteId, record.inventoryDate, record.inventoryTypeCode))<br>      .mapGroups(findLatestRecords).withColumn(&quot;recordTimestamp&quot;,date_format(to_timestamp(col(&quot;recordTimestamp&quot;),UtcTimestampFormat),UtcTimestampFormat)).as[GroupBlockCollationSchema]<br>      .coalesce(5)<br>    val avgBlockedRevenueDsA: Dataset[(GroupBlockRow,String,String, Boolean)] = getLatestRecsDS<br>      .withColumn(GroupBlock,<br>        to_json(struct(ActionIndicator, PropertyCodeConst, GroupCode, RecordTimestamp,<br>          ConfirmationNumber, GroupIndicator, InvDate, InventoryTypeCode, BlockedRooms,<br>          BlockedRevenue, CurrencyCode)))<br>      .withColumn(EventSource, lit(jobName))<br>      .withColumn(EventVersion, lit(EmptyStr))<br>      .withColumn(EventIdentifier, lit(jobIdentifier))<br>      .withColumn(RetryCounter, lit(0))<br>      .withColumn(MetaData, typedLit(Map[String, String]()))<br>      .withColumn(&quot;DestinationTopic&quot;, lit(EmptyStr))<br>      .withColumn(RecordNum, lit(0))<br>      .withColumn(TimeStamp, current_timestamp())<br>      .withColumn(Error, lit(EmptyStr))<br>      .withColumn(ErrorType, EmptyArray())<br>      .withColumn(Message, EmptyArray())<br>      .withColumn(Warning, EmptyArray())<br>      .withColumn(PayLoad, col(GroupBlock))<br>      .withColumn(PayLoadType, lit(Json))<br>      .withColumn(GroupBlockEventJson,<br>        to_json(struct(TimeStamp, MetaData, RetryCounter, PayLoad, EventVersion, EventSource, &quot;DestinationTopic&quot;, EventIdentifier, PayLoadType,<br>          RecordNum, Error, ErrorType, Message, Warning)))<br>      .withColumn(GroupEffDate, when(col(ActionIndicator) === Delete, current_date).otherwise(col(GroupEffectiveDate)))<br>      .withColumn(GroupExpDate, when(col(ActionIndicator) === Delete, current_date).otherwise(col(GroupExpireDate)))<br>      .groupByKey(row =&gt; {<br>        GroupBlockData(<br>          row.getAs(PropertyCodeConst),<br>          row.getAs(GroupCode),<br>          row.getAs[String](ConfirmationNumber),<br>          checkForNull(row.getAs(GroupIndi)),<br>          Some(row.getAs(GroupEffDate)),<br>          Some(row.getAs(GroupExpDate)),<br>          checkForNull(row.getAs(OpportunityId)),<br>          checkForNull(row.getAs(QuoteId)),<br>          checkForNull(row.getAs(GroupEventName)),<br>          getTimestamp(row.getAs(RecordTimestamp), UtcTimestampFormat),<br>          row.getAs(ActionIndicator),<br>          PgbLastUpdBy, row.getAs(srcPayload))<br>      })<br>      .mapGroups(mergeToSingleRecord).coalesce(5)<br>      .map(record =&gt; {<br>        log.info(&quot;avgBlockedRevenueDS :: &quot; + record, logObj)<br>        record<br>      })<br>    invDtLevelDataWithCurCodeDSRecoverable.union(avgBlockedRevenueDsA).coalesce(5)<br>  }<br><br><br>}<br><br><br><br>calendar_data<br><br>package com.marriott.mgpapp.spark.groupblock.collation.udfs<br><br>import java.sql.Date<br>import java.text.SimpleDateFormat<br>import java.util.Calendar<br>import org.apache.spark.sql.functions.udf<br><br>object SparkUdfs {<br><br>  lazy val EmptyArray = udf(() =&gt; Array.empty[Int])<br><br>  //Spark UDF for getting the Amount<br>  lazy val getAmount = udf(getAmt)<br><br>  lazy val getAmt: (Int, Int) =&gt; Double = (amt: Int, dec_prec: Int) =&gt; {<br>    val amount = dec_prec match {<br>      case x if 1 to 4 contains x =&gt; amt / math.pow(10, x)<br>      case _ =&gt; amt<br>    }<br>    amount<br>  }<br><br>  //Spark UDF for getting the list of inventory dates<br>  lazy val getInventoryDateList = udf(getInventoryDtList)<br><br>  lazy val getInventoryDtList: (String, Date, Int) =&gt; Seq[Date] = (verb: String, invStartDt: Date, invDtSpan: Int) =&gt; {<br><br>    if (verb == &quot;D&quot; &amp;&amp; invDtSpan == 0) {<br>      Seq(Date.valueOf(&quot;9999-12-31&quot;))<br>    } else {<br>      val calendar = Calendar.getInstance<br>      val simpleDateFormat = new SimpleDateFormat(&quot;yyyy-MM-dd&quot;)<br><br>      for {i &lt;- 0 until invDtSpan} yield {<br>        calendar.setTime(invStartDt)<br>        calendar.add(Calendar.DATE, i)<br>        Date.valueOf(simpleDateFormat.format(calendar.getTime))<br>      }<br>    }<br>  }<br>}<br><br><br><br><br>collation_test<br><br><br><br><br><br>dao<br>dbreader_test<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.datasource<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.Constants.TGTGroupBlockTable<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbReader<br>import com.marriott.mgpapp.spark.groupblock.collation.exceptions.PostgresException<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.util.BuilderUtil<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.apache.spark.sql.SparkSession<br>import org.mockito.ArgumentMatchers.{any, anyString}<br>import org.mockito.{ArgumentMatchers, MockitoSugar}<br>import org.scalatest.funsuite.AnyFunSuite<br><br>import java.sql.{Connection, PreparedStatement, ResultSet, SQLException, Statement}<br><br>class DbReaderTest  extends AnyFunSuite  with MockitoSugar {<br>  UserGroupInformation.setLoginUser(<br>    UserGroupInformation.createRemoteUser(&quot;hduser&quot;)<br>  )<br><br><br>  implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>  implicit val jobConfigDetails: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>  implicit val logObj :CustomLoggerInfo = new CustomLoggerInfo(&quot;&quot;,&quot;&quot;,&quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>    &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br>  val spark = mock[SparkSession]<br>  val dbDetails = TestsHelper.dbDetails<br>  val schemaMap: Map[String, String] = Map(&quot;key&quot; -&gt; &quot;value&quot;, &quot;TGTGroupBlockTable&quot; -&gt; &quot;TGTGroupBlockTable&quot;, &quot;PropertyTableIN&quot; -&gt; &quot;PropertyTableIN&quot;, &quot;PropParamAssignTableIN&quot; -&gt; &quot;PropParamAssignTableIN&quot;)<br>  val dbSink = new DbReader(schemaMap)<br>  var mockConnection: Connection = _<br>  var pst: Statement = _<br><br>  test(&quot;Open function success case&quot;) {<br>    withObjectMocked[BuilderUtil.type] {<br>      mockConnection = mock[Connection]<br>      when(BuilderUtil.getDBConnection(<br>        ArgumentMatchers.any(classOf[ConnectionConfigDetails])<br>      )(ArgumentMatchers.any())).thenReturn(Option(mockConnection))<br><br>      val openResult = dbSink.open()<br>      assert(openResult)<br>    }<br>  }<br><br><br>  test(&quot;process function&quot;) {<br>    import spark.implicits._<br>    withObjectMocked[BuilderUtil.type] {<br>      mockConnection = mock[Connection]<br>      when(BuilderUtil.getDBConnection(<br>        ArgumentMatchers.any(classOf[ConnectionConfigDetails])<br>      )(ArgumentMatchers.any())).thenReturn(Option(mockConnection))<br>      when(dbSink.open()).thenReturn(true)<br>      //  pst = mock[PreparedStatement]<br>      pst = mock[Statement]<br><br>      val mockResultSet = mock[ResultSet]<br>      val query = &quot;SELECT * FROM test_pj_gb&quot;<br>        when(mockConnection.createStatement()).thenReturn(pst)<br>      when(pst.executeQuery(query)).thenReturn(mockResultSet)<br>      when(mockResultSet.next()).thenReturn(true, false) // Simulate a ResultSet with one row<br>      when(mockResultSet.getString(any[String])).thenReturn(&quot;dummyValue&quot;)<br>      when(mockResultSet.getTimestamp(any[String]))<br>        .thenReturn(java.sql.Timestamp.valueOf(&quot;2022-01-01 00:00:00.0&quot;))<br>      val result = dbSink.processLookupPjGb(query)<br><br>      verify(pst).executeQuery(query)<br>    }<br>  }<br>  test(&quot;processBrandCode function&quot;) {<br>    withObjectMocked[BuilderUtil.type] {<br>      mockConnection = mock[Connection]<br>      when(BuilderUtil.getDBConnection(<br>        ArgumentMatchers.any(classOf[ConnectionConfigDetails])<br>      )(ArgumentMatchers.any())).thenReturn(Option(mockConnection))<br>      when(dbSink.open()).thenReturn(true)<br>      pst = mock[Statement]<br>     val mockResultSet = mock[ResultSet]<br><br>      val query = &quot;SELECT * FROM test_pj_gb&quot;<br>      when(mockConnection.createStatement()).thenReturn(pst)<br>      when(pst.executeQuery(query)).thenReturn(mockResultSet)<br>      when(mockResultSet.next()).thenReturn(true, false)<br><br>      val result = dbSink.processBrandCode(query)<br><br>      verify(pst).executeQuery(query)<br>      assert(result.isDefined)<br>      assert(result.get)<br>    }<br>  }<br><br><br>  test(&quot;processPropParamAssignMap function&quot;) {<br>    withObjectMocked[BuilderUtil.type] {<br>      val mockConnection = mock[Connection]<br>      val mockStatement = mock[Statement]<br>      val mockResultSet = mock[ResultSet]<br>      val PpaPropCode = &quot;PpaPropCode&quot;<br><br><br>      // Define the behavior of the mocks<br>      when(mockConnection.createStatement()).thenReturn(mockStatement)<br>      val query = &quot;SELECT * FROM test_pj_gb&quot;<br>      when(mockStatement.executeQuery(query)).thenReturn(mockResultSet)<br>      when(mockResultSet.next()).thenReturn(true, false) // Simulate a ResultSet with one row<br>      when(mockResultSet.getString(PpaPropCode)).thenReturn(&quot;dummyValue&quot;)<br><br>      // Call the function<br>      val dbSink = new DbReader(schemaMap) // Replace ... with necessary arguments<br>      dbSink.connection = Some(mockConnection) // Set the mockConnection to be used in processPropParamAssignMap<br>      val result = dbSink.processPropParamAssignMap(query)<br><br>      // Verify that the query was executed<br>      verify(mockStatement).executeQuery(query)<br><br>      // Check the result<br>      assert(result.isDefined)<br>      //assert(result.get == Map(PpaPropCode -&gt; &quot;dummyValue&quot;))<br>    }<br>  }<br><br><br><br>  test(&quot;processPropParam function - success scenario&quot;) {<br>    withObjectMocked[BuilderUtil.type] {<br>      val mockConnection = mock[Connection]<br>      val mockStatement = mock[Statement]<br>      val mockResultSet = mock[ResultSet]<br><br>      // Define the behavior of the mocks<br>      when(mockConnection.createStatement()).thenReturn(mockStatement)<br>      val query = &quot;SELECT * FROM test_pj_gb&quot;<br>      when(mockStatement.executeQuery(query)).thenReturn(mockResultSet)<br>      when(mockResultSet.next()).thenReturn(true, false) // Simulate a ResultSet with one row<br>      when(mockResultSet.getString(&quot;ppa_param_value&quot;)).thenReturn(&quot;dummyValue&quot;)<br><br>      // Call the function<br>      val dbSink = new DbReader(schemaMap) // Replace ... with necessary arguments<br>      dbSink.connection = Some(mockConnection) // Set the mockConnection to be used in processPropParam<br>      val result = dbSink.processPropParam(query)<br><br>      // Verify that the query was executed<br>      verify(mockStatement).executeQuery(query)<br><br>      // Check the result<br>      assert(result.isDefined)<br>      assert(result.get == &quot;dummyValue&quot;)<br>    }<br>  }<br><br><br>}<br><br><br><br><br>dbsinktest<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.datasource<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.datasource.PostgresSqlDataSourceWO<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.Constants.appNameForLogger<br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import com.marriott.mgpapp.spark.groupblock.collation.dao.{GroupBlock, GroupBlockRow}<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbSink<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.marriott.mgpapp.spark.groupblock.collation.exceptions.PostgresException<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper._<br>import com.marriott.utility.CustomLogger<br>import com.zaxxer.hikari.HikariDataSource<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.mockito.ArgumentMatchers<br>import org.mockito.scalatest.MockitoSugar<br>import org.scalatest.BeforeAndAfterAll<br>import org.scalatest.funsuite.AnyFunSuite<br>import org.scalatest.matchers.must.Matchers<br>import slick.dbio.{DBIOAction, Effect, NoStream}<br>import slick.jdbc.JdbcBackend.{Database, Session}<br>import slick.lifted.{TableQuery, Tag}<br><br>import java.sql.{Date, Timestamp}<br>import scala.concurrent.Future<br>import scala.language.{existentials, higherKinds, implicitConversions}<br><br>class DbSinkTest<br>    extends AnyFunSuite<br>    with Matchers<br>    with MockitoSugar<br>    with BeforeAndAfterAll<br>    with Serializable {<br>  UserGroupInformation.setLoginUser(<br>    UserGroupInformation.createRemoteUser(&quot;hduser&quot;)<br>  )<br><br>  implicit val jobConfigurationDetails: JobConfigurationDetails =<br>    jobConfigDetails<br>  implicit val connConfigDetails: ConnectionConfigDetails =<br>    TestsHelper.connConfigDetails<br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br>  implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;,&quot;&quot;,&quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>    &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br><br>  var sink: DbSink = _<br>  var mockDataSource: HikariDataSource = _<br>  var mockDatabase: Database = _<br>  var mockSession: Session = _<br><br>  override protected def beforeAll(): Unit = {<br>    mockDataSource = mock[HikariDataSource]<br>    mockDatabase = mock[Database]<br>    mockSession = mock[Session]<br><br>    sink = new DbSink(<br>      connConfigDetails.dbconfig.get,<br>      &quot;test&quot;,<br>      &quot;testtable&quot;,<br>      &quot;collationTest&quot;,<br>      &quot;CollationTest&quot;<br>    )<br>  }<br><br>  test(&quot;DB Sink Open&quot;) {<br>    withObjectMocked[PostgresSqlDataSourceWO.type] {<br>      when(<br>        PostgresSqlDataSourceWO.setConfig(<br>          ArgumentMatchers.any(classOf[DbDetails])<br>        )<br>      ).thenReturn(Right())<br><br>      when(PostgresSqlDataSourceWO.getDataSource())<br>        .thenReturn(Right(mockDataSource))<br><br>      //Positive Scenario<br>      when(mockDatabase.createSession()).thenReturn(mockSession)<br><br>      var openResult = sink.open(<br>        partitionId = 0,<br>        version = 0<br>      )<br>      assert(openResult)<br>    }<br>  }<br><br>  test(&quot;DbSink.process should handle insert/update and delete operations&quot;) {<br>    sink.session = mockSession<br>    sink.GroupBlockHandler = TableQuery[GroupBlock]((tag: Tag) =&gt;<br>      new GroupBlock(tag, Some(schemaName), &quot;group_block_test&quot;)<br>    )<br><br>    val data = Seq(<br>      (<br>        GroupBlockRow(<br>          pgb_prop_code = &quot;CHIRZ&quot;,<br>          pgb_group_code = &quot;GRP12&quot;,<br>          pgb_cnfrm_num = Some(&quot;13295284&quot;),<br>          pgb_grp_ctrct_ind = &quot;G&quot;,<br>          pgb_start_dt = Some(Date.valueOf(&quot;2023-12-22&quot;)),<br>          pgb_end_dt = Some(Date.valueOf(&quot;2023-12-12&quot;)),<br>          pgb_oppty_id = &quot;ABC&quot;,<br>          pgb_quote_id = &quot;1-ABC&quot;,<br>          pgb_group_name = &quot;Group Contract Name&quot;,<br>          pgb_transmission_ts = Some(Timestamp.valueOf(&quot;2019-12-21 23:59:50&quot;)),<br>          pgb_verb = &quot;D&quot;,<br>          pgb_last_up_by = &quot;GROUP BLOCK COLLATION JOB&quot;,<br>          pgb_last_up_ts = Timestamp.valueOf(&quot;2023-12-20 18:08:34.089&quot;)<br>        ),<br>        &quot;testString&quot;,&quot;&quot;,false<br>      ),<br>      (<br>        GroupBlockRow(<br>          pgb_prop_code = &quot;CHIRZ&quot;,<br>          pgb_group_code = &quot;GRP12&quot;,<br>          pgb_cnfrm_num = Some(&quot;13295284&quot;),<br>          pgb_grp_ctrct_ind = &quot;G&quot;,<br>          pgb_start_dt = Some(Date.valueOf(&quot;2023-12-22&quot;)),<br>          pgb_end_dt = Some(Date.valueOf(&quot;2023-12-12&quot;)),<br>          pgb_oppty_id = &quot;ABC&quot;,<br>          pgb_quote_id = &quot;1-ABC&quot;,<br>          pgb_group_name = &quot;Group Contract Name&quot;,<br>          pgb_transmission_ts = Some(Timestamp.valueOf(&quot;2019-12-21 23:59:50&quot;)),<br>          pgb_verb = &quot;DS&quot;,<br>          pgb_last_up_by = &quot;GROUP BLOCK COLLATION JOB&quot;,<br>          pgb_last_up_ts = Timestamp.valueOf(&quot;2023-12-20 18:08:34.089&quot;)<br>        ),<br>        &quot;testString&quot;,&quot;&quot;,false<br>      ),<br>      (<br>        GroupBlockRow(<br>          pgb_prop_code = &quot;CHIRZ&quot;,<br>          pgb_group_code = &quot;GRP12&quot;,<br>          pgb_cnfrm_num = Some(&quot;13295284&quot;),<br>          pgb_grp_ctrct_ind = &quot;G&quot;,<br>          pgb_start_dt = Some(Date.valueOf(&quot;2023-12-22&quot;)),<br>          pgb_end_dt = Some(Date.valueOf(&quot;2023-12-12&quot;)),<br>          pgb_oppty_id = &quot;ABC&quot;,<br>          pgb_quote_id = &quot;1-ABC&quot;,<br>          pgb_group_name = &quot;Group Contract Name&quot;,<br>          pgb_transmission_ts = Some(Timestamp.valueOf(&quot;2019-12-21 23:59:50&quot;)),<br>          pgb_verb = &quot;C&quot;,<br>          pgb_last_up_by = &quot;GROUP BLOCK COLLATION JOB&quot;,<br>          pgb_last_up_ts = Timestamp.valueOf(&quot;2023-12-20 18:08:34.089&quot;)<br>        ),<br>        &quot;testString&quot;,&quot;&quot;,false<br>      )<br>    )<br>    when(mockSession.database).thenReturn(mockDatabase)<br><br>    when(<br>      mockDatabase.run(<br>        ArgumentMatchers.any(<br>          classOf[<br>            DBIOAction[AnyVal, NoStream, Effect.Write]<br>          ]<br>        )<br>      )<br>    ).thenReturn(Future.successful(1))<br><br>    try {<br>      for (row &lt;- data)<br>        sink.process(row)<br>    } catch {<br>      case e: Exception =&gt;<br>        log.error(&quot;Exception occured for positive scenario&quot;, logObj)<br>    }<br>    assert(true)<br><br>    when(<br>      mockDatabase.run(<br>        ArgumentMatchers.any(<br>          classOf[<br>            DBIOAction[AnyVal, NoStream, Effect.Write]<br>          ]<br>        )<br>      )<br>    ).thenReturn(Future.failed(new Exception(&quot;Error Occurred&quot;)))<br><br>    intercept[PostgresException] {<br>      for (row &lt;- data)<br>        sink.process(row)<br>    }<br>  }<br><br>  test(&quot;DB Close Test&quot;) {<br>    sink.session = mockSession<br><br>    doNothing.when(mockSession).close()<br>    try {<br>      sink.close(null)<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Error Occurred for DB close connect&quot;, logObj)<br>    }<br>    assert(true)<br><br>    intercept[PostgresException] {<br>      sink.close(new Exception(&quot;Dummy Exception&quot;))<br>    }<br>  }<br><br>}<br><br><br><br><br>datasource<br><br><br>entity<br><br>groupblockentity.scala<br>package com.marriott.mgpapp.spark.groupblock.collation.test.entity<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.{GroupBlockCollationSchema, GroupBlockEntityAdj, GroupBlockEntityNew, GroupBlockEntityNewUpdated, GroupBlockEntityPropCode, GroupBlockEntityUpdatedCurrencyCode, ReservationGroupBlockMerge}<br>import org.scalatest.funsuite.AnyFunSuite<br><br> import java.sql.Date<br><br>  class GroupBlockEntityTest extends AnyFunSuite {<br><br>    val date = new Date(System.currentTimeMillis())<br><br><br>    test(&quot;GroupBlockEntityPropCode should be valid&quot;) {<br><br>      val propCodeSample = GroupBlockEntityPropCode(<br>        Some(&quot;A&quot;), Some(&quot;timestamp&quot;), Some(&quot;oppId&quot;), Some(&quot;quoteId&quot;), Some(&quot;confNo&quot;),<br>        Some(&quot;propCode&quot;), Some(&quot;groupCode&quot;), Some(date), Some(date), Some(&quot;eventName&quot;),<br>        Some(&quot;eventType&quot;), Some(&quot;invTypeCode&quot;), Some(&quot;ratePlanCode&quot;), Some(date), Some(date),<br>        Some(123), Some(&quot;currencyCode&quot;), Some(2), Some(10), Some(date), Some(date),<br>        &quot;ppaPropCode&quot;, &quot;payload&quot;, false<br><br>      )<br>      assert(propCodeSample.actionIndicator == Some(&quot;A&quot;))<br>    }<br><br>    test(&quot;GroupBlockEntityAdj should be valid&quot;) {<br><br>      val adjSample = GroupBlockEntityAdj(<br>        &quot;A&quot;, &quot;timestamp&quot;, &quot;oppId&quot;, &quot;quoteId&quot;, &quot;confNo&quot;,<br>        &quot;propCode&quot;, &quot;groupCode&quot;, date, date, &quot;eventName&quot;,<br>        &quot;eventType&quot;, &quot;invTypeCode&quot;, &quot;ratePlanCode&quot;, date, date,<br>        Some(123), &quot;currencyCode&quot;, Some(2), Some(10), date, date,<br>        &quot;ppaPropCode&quot;, &quot;dataSource&quot;, &quot;groupIndicator&quot;, &quot;groupSourceIndicator&quot;,<br>        5, &quot;inventoryDate&quot;, 123.45, &quot;additionalIndicator&quot;<br>      )<br><br>      assert(adjSample.actionIndicator == &quot;A&quot;)<br>    }<br><br><br>    test(&quot;GroupBlockEntityNew should be valid&quot;) {<br><br>      val currentTimestamp = new Date(System.currentTimeMillis())<br>      val sampleEntity = GroupBlockEntityNew(<br>        &quot;A&quot;, &quot;timestamp&quot;, &quot;oppId&quot;, &quot;quoteId&quot;, &quot;confNo&quot;,<br>        &quot;propCode&quot;, &quot;groupCode&quot;, currentTimestamp, currentTimestamp, &quot;eventName&quot;,<br>        &quot;eventType&quot;, &quot;invTypeCode&quot;, &quot;ratePlanCode&quot;, currentTimestamp, currentTimestamp,<br>        Some(123), &quot;currencyCode&quot;, Some(2), Some(10), currentTimestamp, currentTimestamp,<br>        &quot;ppaPropCode&quot;, &quot;dataSource&quot;, &quot;groupIndicator&quot;, &quot;groupSourceIndicator&quot;,<br>        5, currentTimestamp, 123.45, &quot;payload&quot;, isRecoverable = false<br>      )<br><br>      assert(sampleEntity.actionIndicator == &quot;A&quot;)<br>      // Add more assertions as needed<br>    }<br><br><br><br>    test(&quot;GroupBlockEntityNew apply method should be valid&quot;) {<br><br>      val samplePayload = &quot;samplePayload&quot;<br>      val sampleEntity = GroupBlockEntityNew(samplePayload, isRecoverable = true)<br><br>      assert(sampleEntity.actionIndicator == &quot;&quot;)<br>      assert(sampleEntity.transmissionTimestamp == &quot;&quot;)<br>      assert(sampleEntity.opportunityId == &quot;&quot;)<br>      assert(sampleEntity.quoteId == &quot;&quot;)<br>      assert(sampleEntity.confirmationNumber == &quot;&quot;)<br>      assert(sampleEntity.propertyCode == &quot;&quot;)<br>      assert(sampleEntity.groupCode == &quot;&quot;)<br>      assert(sampleEntity.groupEventName == &quot;&quot;)<br>      assert(sampleEntity.groupEventType == &quot;&quot;)<br>      assert(sampleEntity.inventoryTypeCode == &quot;&quot;)<br>      assert(sampleEntity.ratePlanCode == &quot;&quot;)<br>      assert(sampleEntity.amount.isEmpty)<br>      assert(sampleEntity.currencyCode == &quot;&quot;)<br>      assert(sampleEntity.decimalPrecision.isEmpty)<br>      assert(sampleEntity.blockedRooms.isEmpty)<br>      assert(sampleEntity.ppa_prop_code == &quot;&quot;)<br>      assert(sampleEntity.data_source_ind == &quot;&quot;)<br>      assert(sampleEntity.groupIndicator == &quot;&quot;)<br>      assert(sampleEntity.groupSourceIndicator == &quot;&quot;)<br>      assert(sampleEntity.InvDateSpan == 0)<br>      assert(sampleEntity.Amt == 0.0)<br>      assert(sampleEntity.payload == samplePayload)<br>      assert(sampleEntity.isRecoverable)<br>    }<br>    test(&quot;GroupBlockEntityNewUpdated apply method should be valid&quot;) {<br><br>      val samplePayload = &quot;samplePayload&quot;<br>      val sampleEntity = GroupBlockEntityNewUpdated(samplePayload, isRecoverable = true)<br><br>      assert(sampleEntity.actionIndicator == &quot;&quot;)<br>      assert(sampleEntity.transmissionTimestamp == &quot;&quot;)<br>      assert(sampleEntity.opportunityId == &quot;&quot;)<br>      assert(sampleEntity.quoteId == &quot;&quot;)<br>      assert(sampleEntity.confirmationNumber == &quot;&quot;)<br>      assert(sampleEntity.propertyCode == &quot;&quot;)<br>      assert(sampleEntity.groupCode == &quot;&quot;)<br>      assert(sampleEntity.groupEventName == &quot;&quot;)<br>      assert(sampleEntity.groupEventType == &quot;&quot;)<br>      assert(sampleEntity.inventoryTypeCode == &quot;&quot;)<br>      assert(sampleEntity.ratePlanCode == &quot;&quot;)<br>      assert(sampleEntity.amount.isEmpty)<br>      assert(sampleEntity.currencyCode == &quot;&quot;)<br>      assert(sampleEntity.decimalPrecision.isEmpty)<br>      assert(sampleEntity.blockedRooms.isEmpty)<br>      assert(sampleEntity.ppa_prop_code == &quot;&quot;)<br>      assert(sampleEntity.data_source_ind == &quot;&quot;)<br>      assert(sampleEntity.groupIndicator == &quot;&quot;)<br>      assert(sampleEntity.groupSourceIndicator == &quot;&quot;)<br>      assert(sampleEntity.Amt == 0.0)<br>      assert(sampleEntity.payload == samplePayload)<br>      assert(sampleEntity.isRecoverable)<br>    }<br><br><br>    test(&quot;GroupBlockEntityUpdatedCurrencyCode apply method should be valid&quot;) {<br><br>      val samplePayload = &quot;samplePayload&quot;<br>      val sampleEntity = GroupBlockEntityUpdatedCurrencyCode(samplePayload, isRecoverable = true)<br><br>      assert(sampleEntity.actionIndicator == &quot;&quot;)<br>      assert(sampleEntity.transmissionTimestamp == &quot;&quot;)<br>      assert(sampleEntity.opportunityId == &quot;&quot;)<br>      assert(sampleEntity.quoteId == &quot;&quot;)<br>      assert(sampleEntity.confirmationNumber == &quot;&quot;)<br>      assert(sampleEntity.propertyCode == &quot;&quot;)<br>      assert(sampleEntity.groupCode == &quot;&quot;)<br>      assert(sampleEntity.groupEventName == &quot;&quot;)<br>      assert(sampleEntity.groupEventType == &quot;&quot;)<br>      assert(sampleEntity.inventoryTypeCode == &quot;&quot;)<br>      assert(sampleEntity.ratePlanCode == &quot;&quot;)<br>      assert(sampleEntity.amount.isEmpty)<br>      assert(sampleEntity.currencyCode == &quot;&quot;)<br>      assert(sampleEntity.decimalPrecision.isEmpty)<br>      assert(sampleEntity.blockedRooms.isEmpty)<br>      assert(sampleEntity.groupIndicator == &quot;&quot;)<br>      assert(sampleEntity.groupSourceIndicator == &quot;&quot;)<br>      assert(sampleEntity.Amt == 0.0)<br>      assert(sampleEntity.payload == samplePayload)<br>      assert(sampleEntity.isRecoverable)<br>    }<br><br>    test(&quot;ReservationGroupBlockMerge should be valid&quot;) {<br><br>      val sampleEntity = ReservationGroupBlockMerge(<br>        Some(&quot;timeStamp&quot;),<br>        Map(&quot;metadata1&quot; -&gt; Some(&quot;value1&quot;), &quot;metadata2&quot; -&gt; Some(&quot;value2&quot;)),<br>        Some(0),<br>        Some(&quot;payLoad&quot;),<br>        Some(&quot;eventVersion&quot;),<br>        Some(&quot;eventSource&quot;),<br>        Some(&quot;destinationTopic&quot;),<br>        Some(&quot;eventIdentifier&quot;),<br>        Some(&quot;payLoadType&quot;),<br>        Some(&quot;recordNum&quot;),<br>        Some(&quot;error&quot;),<br>        Seq(Some(&quot;errorType1&quot;), Some(&quot;errorType2&quot;)),<br>        Seq(Some(&quot;message1&quot;), Some(&quot;message2&quot;)),<br>        Seq(Some(&quot;warning1&quot;), Some(&quot;warning2&quot;))<br>      )<br><br>      assert(sampleEntity.timeStamp.contains(&quot;timeStamp&quot;))<br>      assert(sampleEntity.metadata == Map(&quot;metadata1&quot; -&gt; Some(&quot;value1&quot;), &quot;metadata2&quot; -&gt; Some(&quot;value2&quot;)))<br>      assert(sampleEntity.retryCounter.contains(0))<br>      assert(sampleEntity.payLoad.contains(&quot;payLoad&quot;))<br>      assert(sampleEntity.eventVersion.contains(&quot;eventVersion&quot;))<br>      assert(sampleEntity.eventSource.contains(&quot;eventSource&quot;))<br>      assert(sampleEntity.destinationTopic.contains(&quot;destinationTopic&quot;))<br>      assert(sampleEntity.eventIdentifier.contains(&quot;eventIdentifier&quot;))<br>      assert(sampleEntity.payLoadType.contains(&quot;payLoadType&quot;))<br>      assert(sampleEntity.recordNum.contains(&quot;recordNum&quot;))<br>      assert(sampleEntity.error.contains(&quot;error&quot;))<br>      assert(sampleEntity.errorType == Seq(Some(&quot;errorType1&quot;), Some(&quot;errorType2&quot;)))<br>      assert(sampleEntity.message == Seq(Some(&quot;message1&quot;), Some(&quot;message2&quot;)))<br>      assert(sampleEntity.warning == Seq(Some(&quot;warning1&quot;), Some(&quot;warning2&quot;)))<br>    }<br>    test(&quot;GroupBlockCollationSchema apply method should be valid&quot;) {<br><br>      val samplePayload = &quot;samplePayload&quot;<br>      val sampleEntity = GroupBlockCollationSchema(samplePayload, isRecoverable = true)<br>      assert(sampleEntity.isRecoverable)<br>    }<br><br><br><br><br>}<br>exceptions<br>helper_previous<br>helper_new<br><br>filtration:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.helper<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.Constants.appNameForLogger<br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.{GroupBlockCollation, GroupBlockCollationEvent}<br>import com.marriott.mgpapp.spark.groupblock.collation.helper.GroupBlockFiltrationHelper<br>//import com.marriott.mgpapp.spark.groupblock.collation.jobs.Main.connConfig<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper._<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql.types.{StringType, StructField, StructType}<br>import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}<br>import org.scalatest.BeforeAndAfterEach<br>import org.scalatest.funsuite.AnyFunSuite<br><br>import scala.io.Source<br><br>class GroupBlockFiltrationHelperTest<br>    extends AnyFunSuite<br>    with BeforeAndAfterEach<br>    with GroupBlockFiltrationHelper {<br><br>  // Create a SparkSession<br>  implicit val spark: SparkSession = SparkSession.builder<br>    .appName(groupBlockFiltrationHelperTest)<br>    .master(local)<br>    .getOrCreate()<br>  implicit val jobConfigurationDetails: JobConfigurationDetails =<br>    jobConfigDetails<br>  implicit val connConfig: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>  override val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  override val kafkaTopicMap: Option[Map[String, String]] =<br>    jobConfigDetails.kafkatopicmap<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br>  import spark.implicits._<br>//failing in jenkins but running locally<br>//  test(&quot;groupBlockFiltration should return the correct Dataset&quot;) {<br>//<br>//<br>//    val schema = StructType(<br>//      List(<br>//        StructField(testColumn, StringType, nullable = true)<br>//      )<br>//    )<br>//<br>//    // Create a DataFrame with specific input data<br>//    val data: DataFrame = spark.createDataFrame(<br>//      spark.sparkContext.parallelize(<br>//        Seq(<br>//          Row(testData),<br>//          Row(testData),<br>//          Row(testData)<br>//        )<br>//      ),<br>//      schema<br>//    )<br>//    // Call the function<br>//    val result = groupBlockFiltration(data)<br>//<br>//    // Check the result<br>//    assert(result.isInstanceOf[Dataset[GroupBlockCollationEvent]])<br>//  }<br><br>  test(<br>    &quot;convertToEntityGroupBlockCollation should return GroupBlockCollation when JSON is valid&quot;<br>  ) {<br>    val validInputFilePath =<br>      &quot;./GroupBlockCollation/src/test/resources/validInput.json&quot;<br>    val entitygood: String = Source.fromFile(validInputFilePath).mkString<br>    val result = convertToEntityGroupBlockCollation(entitygood)(spark)<br>    assert(result._1.isDefined)<br>    assert(result.isInstanceOf[(Option[GroupBlockCollation], Option[String])])<br>  }<br><br>  ignore(<br>    &quot;convertToEntityGroupBlockCollation should return GroupBlockCollationEvent when JSON is invalid&quot;<br>  ) {<br>    val invalidInputFilePath =<br>      &quot;GroupBlockCollation/src/test/resources/invalidInput.json&quot;<br>    val entitybad: String = Source.fromFile(invalidInputFilePath).mkString<br>    val result = convertToEntityGroupBlockCollation(entitybad)(spark)<br>    //assert(result._1.isEmpty)<br>    assert(result.isInstanceOf[(Option[GroupBlockCollation], Option[String])])<br>  }<br><br>  test(&quot;transformtoDF - positive scenario&quot;) {<br>    val validInputFilePath =<br>      &quot;GroupBlockCollation/src/test/resources/validInput.json&quot;<br>    val entitygood: String = Source.fromFile(validInputFilePath).mkString<br>    val gbDataValid = convertToEntityGroupBlockCollation(entitygood)(spark)<br>    val data = Seq(gbDataValid).toDF(&quot;value&quot;, &quot;payload&quot;).drop(&quot;payload&quot;)<br>    val result = transformtoDF(data)<br>    assert(result.count() == 1)<br>    assert(result.columns.contains(&quot;HotelId&quot;))<br>  }<br><br>  test(&quot;convertToFinalDS should return the correct Dataset&quot;) {<br>    import spark.implicits._<br>    val validInputFilePath =<br>      &quot;GroupBlockCollation/src/test/resources/validInput.json&quot;<br>    val entitygood: String = Source.fromFile(validInputFilePath).mkString<br>    val gbDataValid = convertToEntityGroupBlockCollation(entitygood)(spark)<br>    val data = Seq(gbDataValid).toDF(&quot;value&quot;, &quot;payload&quot;).drop(&quot;payload&quot;)<br>    val gbValidTransform = transformtoDF(data)<br>    val result = gbValidTransform.map(row =&gt; convertToFinalDS(row))<br>    assert(result.isInstanceOf[Dataset[GroupBlockCollationEvent]])<br>  }<br><br>  test(<br>    &quot;convertToFinalDS should convert a valid Row to GroupBlockCollationEvent&quot;<br>  ) {<br><br>    // Define a valid Row<br>    val schema = StructType(<br>      Seq(<br>        StructField(&quot;payload&quot;, StringType, nullable = true),<br>        StructField(&quot;opportunityId&quot;, StringType, nullable = true),<br>        StructField(&quot;quoteId&quot;, StringType, nullable = true),<br>        StructField(&quot;actionEnum&quot;, StringType, nullable = true),<br>        StructField(&quot;generationTime&quot;, StringType, nullable = true),<br>        StructField(&quot;confirmationNumber&quot;, StringType, nullable = true),<br>        StructField(&quot;propertyCode&quot;, StringType, nullable = true),<br>        StructField(&quot;groupCode&quot;, StringType, nullable = true),<br>        StructField(&quot;groupName&quot;, StringType, nullable = true),<br>        StructField(&quot;groupType&quot;, StringType, nullable = true),<br>        StructField(&quot;groupEffectiveDate&quot;, StringType, nullable = true),<br>        StructField(&quot;groupExpireDate&quot;, StringType, nullable = true),<br>        StructField(&quot;ratePlanEffectiveDate&quot;, StringType, nullable = true),<br>        StructField(&quot;ratePlanExpireDate&quot;, StringType, nullable = true),<br>        StructField(&quot;inventoryTypeCode&quot;, StringType, nullable = true),<br>        StructField(&quot;ratePlanCode&quot;, StringType, nullable = true),<br>        StructField(&quot;inventoryEffectiveDate&quot;, StringType, nullable = true),<br>        StructField(&quot;inventoryExpireDate&quot;, StringType, nullable = true),<br>        StructField(&quot;verb&quot;, StringType, nullable = true),<br>        StructField(&quot;amount&quot;, StringType, nullable = true),<br>        StructField(&quot;currencyCode&quot;, StringType, nullable = true),<br>        StructField(&quot;dec_pre&quot;, StringType, nullable = true),<br>        StructField(&quot;blocked&quot;, StringType, nullable = true),<br>        StructField(&quot;HotelId&quot;, StringType, nullable = true)<br>      )<br>    )<br>    val validRow = Row(<br>      &quot;&quot;,<br>      &quot;aoppor&quot;,<br>      &quot;aquote&quot;,<br>      &quot;Create&quot;,<br>      &quot;2023-12-26T23:59:54.000Z&quot;,<br>      &quot;13295284&quot;,<br>      &quot;MEMWI&quot;,<br>      &quot;GA4&quot;,<br>      &quot;Group Contract Name&quot;,<br>      &quot;Convention&quot;,<br>      &quot;2023-12-23&quot;,<br>      &quot;2023-12-25&quot;,<br>      &quot;2023-12-20T23:59:59&quot;,<br>      &quot;2023-12-26T23:59:59&quot;,<br>      &quot;KING&quot;,<br>      null,<br>      &quot;2019-11-22&quot;,<br>      &quot;2019-12-20&quot;,<br>      &quot;C&quot;,<br>      &quot;110&quot;,<br>      &quot;USD&quot;,<br>      &quot;0&quot;,<br>      &quot;0&quot;,<br>      &quot;MEMWIGA413295284&quot;<br>    )<br>    // Define SparkSession for testing<br>    val sparkSS: SparkSession =<br>      SparkSession.builder.appName(&quot;sample&quot;).master(local).getOrCreate()<br>    val row = sparkSS.sparkContext.parallelize(Seq(validRow))<br>    val data = sparkSS.createDataFrame(row, schema)<br><br>    // Call the conversion function<br>    val result = convertToFinalDS(data.first())<br><br>    // Assert that the result is of type GroupBlockCollationEvent and any other necessary assertions<br>    assert(result.isInstanceOf[GroupBlockCollationEvent])<br>  }<br>  ignore(&quot;go to handleMalFormedEvents when JSON is invalid2&quot;) {<br>    import spark.implicits<br>      implicit val  connectionConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>    implicit val  jobConfigDetails: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>    val invalidInputFilePath = &quot;GroupBlockCollation/src/test/resources/invalidInput.json&quot;<br>    val entitybad: String = Source.fromFile(invalidInputFilePath).mkString<br>    val entityDF = spark.createDataset(Seq(entitybad))<br><br>    handleMalFormedEvents(entityDF.toDF())(spark,jobConfigDetails,connectionConfigDetails)<br>    // Assert<br>    // Since handleMalFormedEvents is a Unit method, you cannot assert on its result.<br>  }<br><br>}<br><br><br>helpertest:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.helper<br><br>import com.marriott.elevate.spark.framework.cdc.{CUDBaseTable, CUDPayload}<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.EmptyString<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.Constants.{EmptyStr, appNameForLogger}<br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import com.marriott.mgpapp.spark.groupblock.collation.datasource.DbReader<br>import com.marriott.mgpapp.spark.groupblock.collation.entity._<br>import com.marriott.mgpapp.spark.groupblock.collation.helper.GroupBlockHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper._<br>import com.marriott.mgpapp.spark.groupblock.collation.util.BuilderUtil<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql._<br>import org.mockito.ArgumentMatchers.{any, anyString}<br>import org.mockito.ArgumentMatchersSugar.eqTo<br>import org.mockito.{ArgumentMatchers, MockitoSugar}<br>import org.scalatest.BeforeAndAfterEach<br>import org.scalatest.funsuite.AnyFunSuite<br>import org.scalatest.matchers.must.Matchers.{be, contain, have}<br>import org.scalatest.matchers.should.Matchers.{a, convertToAnyShouldWrapper}<br>import org.mockito.ArgumentMatchers._<br>import org.mockito.{MockedStatic, Mockito}<br>import java.sql._<br>import scala.util.{Failure, Success, Try}<br><br>class GroupBlockHelperTest<br>  extends AnyFunSuite<br>    with BeforeAndAfterEach<br>    with MockitoSugar {<br><br>  var groupBlockHelper: GroupBlockHelper = _<br>  implicit var spark: SparkSession = _<br>  var sqlContext: SQLContext = _<br>  var read: DataFrameReader = _<br>  var dbDetails: DbDetails = _<br>  var connection: Connection = _<br><br>  var statement: Statement = _<br>  var resultSet: ResultSet = _<br>  var GroupBlockEntityDataset: GroupBlockEntity = _<br>  implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>  implicit val jobConfig: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>  implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;,&quot;&quot;,&quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>    &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br>  override def beforeEach(): Unit = {<br>    groupBlockHelper = new GroupBlockHelper {<br>      override val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>      override val schemaMap: Map[String, String] =<br>        jobConfigDetails.schemamap.get ++ connConfigDetails.schemaconfig.get<br>      val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br>      override val jobName: String = &quot;GroupBlockCollation&quot;<br>      override val jobIdentifier: String = &quot;GroupBlock&quot;<br>    }<br>    spark = mock[SparkSession]<br>    sqlContext = mock[SQLContext]<br>    read = mock[DataFrameReader]<br>    dbDetails = DbDetails(<br>      jdbcurl = &quot;jdbc:postgresql://abc.com:5432/&quot;,<br>      drivername = &quot;org.postgresql.Driver&quot;,<br>      username = &quot;abcde&quot;,<br>      password = &quot;password&quot;,<br>      poolsize = poolsize,<br>      idletimeout = 1L,<br>      querytimeout = null,<br>      jdbcurlro = dbUrl,<br>      drivernamero = dbDrivername,<br>      usernamero = dbUsername,<br>      passwordro = dbPassword,<br>      poolsizero = poolsize,<br>      idletimeoutro = 1L,<br>      querytimeoutro = null,<br>      jdbcurlwo = dbUrl,<br>      drivernamewo = dbDrivername,<br>      usernamewo = dbUsername,<br>      passwordwo = dbPassword,<br>      poolsizewo = poolsize,<br>      idletimeoutwo = 1L,<br>      querytimeoutwo = null<br>    )<br>    connection = mock[Connection]<br>    statement = mock[Statement]<br>    resultSet = mock[ResultSet]<br><br>    when(spark.sqlContext).thenReturn(sqlContext)<br>    when(sqlContext.read).thenReturn(read)<br>  }<br><br>  val gbCollationMock1: GroupBlockCollationSchema = GroupBlockCollationSchema(<br>    groupCode,<br>    propertyCode,<br>    confirmationNumber,<br>    null,<br>    null,<br>    groupInd,<br>    groupIndicator,<br>    opportunityId,<br>    quoteId,<br>    groupEventName,<br>    &quot;2022-01-01T12:00:00Z&quot;,<br>    currencyCode,<br>    actionIndicator,<br>    null,<br>    inventoryTypeCode,<br>    1,<br>    23.4,<br>    5L,<br>    recordTimestamp,<br>    67.8<br>  )<br>  val gbCollationMock2: GroupBlockCollationSchema = GroupBlockCollationSchema(<br>    groupCode,<br>    propertyCode,<br>    confirmationNumber,<br>    null,<br>    null,<br>    groupInd,<br>    groupIndicator,<br>    opportunityId,<br>    quoteId,<br>    groupEventName,<br>    &quot;2022-02-02T12:00:00Z&quot;,<br>    currencyCode,<br>    actionIndicator,<br>    null,<br>    inventoryTypeCode,<br>    8,<br>    56.7,<br>    4L,<br>    recordTimestamp,<br>    12.3<br>  )<br><br>  test(&quot;findLatestRecords should return the latest record&quot;) {<br>    val groupedKeyValue = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      &quot;key4&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key6&quot;<br>    )<br>    val record1 = gbCollationMock1<br>    val record2 = gbCollationMock2<br>    val records = List(record1, record2).iterator<br><br>    val result =<br>      Try(groupBlockHelper.findLatestRecords(groupedKeyValue, records))<br><br>    result match {<br>      case Success(value) =&gt; assert(value == record2)<br>      case Failure(e: NoSuchElementException) =&gt;<br>        assert(e.getMessage == &quot;None.get&quot;)<br>      case _ =&gt; fail(&quot;Unexpected exception type&quot;)<br>    }<br>  }<br><br>  test(&quot;findLatestRecords should handle empty record list&quot;) {<br>    val groupedKeyValue = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      &quot;key4&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key6&quot;<br>    )<br>    val records = List.empty[GroupBlockCollationSchema].iterator<br><br>    val result =<br>      Try(groupBlockHelper.findLatestRecords(groupedKeyValue, records))<br><br>    result match {<br>      case Success(value) =&gt;<br>        fail(&quot;Expected NoSuchElementException for empty record list&quot;)<br>      case Failure(e: NoSuchElementException) =&gt;<br>        assert(e.getMessage == &quot;None.get&quot;)<br>      case Failure(e: UnsupportedOperationException) =&gt;<br>        assert(e.getMessage == &quot;empty.reduceLeft&quot;)<br>      case _ =&gt; fail(&quot;Unexpected exception type&quot;)<br>    }<br>  }<br><br>  test(<br>    &quot;findLatestRecords should return the single record when there is only one record&quot;<br>  ) {<br>    val groupedKeyValue = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      &quot;key4&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key6&quot;<br>    )<br>    val record1 = gbCollationMock1<br>    val records = List(record1).iterator<br><br>    val result =<br>      Try(groupBlockHelper.findLatestRecords(groupedKeyValue, records))<br><br>    result match {<br>      case Success(value) =&gt; assert(value == record1)<br>      case Failure(e)     =&gt; fail(s&quot;Unexpected exception: $e&quot;)<br>      case _              =&gt; fail(&quot;Unexpected exception type&quot;)<br>    }<br>  }<br><br>  test(<br>    &quot;findLatestRecords should return one of the records when two records have the same timestamp&quot;<br>  ) {<br>    val groupedKeyValue = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      &quot;key4&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key6&quot;<br>    )<br>    val record1 = gbCollationMock1.copy(transmissionTimestamp =<br>      &quot;2022-02-02T12:00:00Z&quot;<br>    ) // make the timestamps the same<br>    val record2 = gbCollationMock2<br>    val records = List(record1, record2).iterator<br><br>    val result =<br>      Try(groupBlockHelper.findLatestRecords(groupedKeyValue, records))<br><br>    result match {<br>      case Success(value) =&gt;<br>        assert(<br>          value == record1 || value == record2<br>        ) // it could be either record1 or record2<br>      case Failure(e) =&gt; fail(s&quot;Unexpected exception: $e&quot;)<br>      case _          =&gt; fail(&quot;Unexpected exception type&quot;)<br>    }<br>  }<br><br>  test(&quot;findLatestRecords should throw NoSuchElementException&quot;) {<br>    val groupedKeyValue = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      &quot;key4&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key6&quot;<br>    )<br>    val record1 = mock[GroupBlockCollationSchema]<br>    val record2 = mock[GroupBlockCollationSchema]<br>    val records = List(record1, record2).iterator<br><br>    val result =<br>      Try(groupBlockHelper.findLatestRecords(groupedKeyValue, records))<br><br>    result match {<br>      case Success(value) =&gt; assert(value == record2)<br>      case Failure(e: NoSuchElementException) =&gt;<br>        assert(e.getMessage == &quot;None.get&quot;)<br>      case _ =&gt; fail(&quot;Unexpected exception type&quot;)<br>    }<br>  }<br><br>  test(&quot;mapGroupBlockList should update columnMap with keyValuePair&quot;) {<br>    val keyValuePair = Map(&quot;key&quot; -&gt; &quot;value&quot;)<br>    val columnMap = Map(&quot;column1&quot; -&gt; &quot;value1&quot;)<br>    val groupBlockCUD = CUDPayload(null, columnMap, &quot;&quot;, &quot;&quot;)<br>    val groupBlockPayload = GBCollationPayload(null, null, null, groupBlockCUD)<br>    val event = GroupBlockCollationEvent(null, groupBlockPayload)<br><br>    val result =<br>      groupBlockHelper.mapGroupBlockList(Iterator(event), keyValuePair)<br><br>    assert(result.head.getPayload.groupBlockCUD.columnMap.contains(&quot;key&quot;))<br>    assert(result.head.getPayload.groupBlockCUD.columnMap(&quot;key&quot;) == &quot;value&quot;)<br>  }<br><br>  test(<br>    &quot;Positive: filterPhMSTsList should filter GroupBlockCollationEvent based on timestamps&quot;<br>  ) {<br>    val loadTimeStamp = &quot;2019-12-21T23:59:50Z&quot;<br>    val pgbTransmissionTs = &quot;2022-03-15 10:15:31.123&quot;<br>    val columnMap = Map(&quot;pgb_transmission_ts&quot; -&gt; pgbTransmissionTs)<br>    val baseTable = CUDBaseTable(&quot;&quot;, &quot;&quot;, &quot;&quot;, loadTimeStamp, &quot;&quot;)<br>    val groupBlockCUD = CUDPayload(baseTable, columnMap, &quot;&quot;, &quot;&quot;)<br>    val groupBlockPayload = GBCollationPayload(null, null, null, groupBlockCUD)<br>    val event = GroupBlockCollationEvent(null, groupBlockPayload)<br><br>    val gbList = List(event)<br><br>    val result = groupBlockHelper.filterPhMSTsList(gbList)<br><br>    assert(result.isEmpty)<br>  }<br><br>  test(<br>    &quot;Negative: filterPhMSTsList should not filter GroupBlockCollationEvent based on timestamps&quot;<br>  ) {<br>    val loadTimeStamp = &quot;2022-12-21T23:59:50Z&quot;<br>    val pgbTransmissionTs = &quot;2022-03-15 10:15:31.123&quot;<br>    val columnMap = Map(&quot;pgb_transmission_ts&quot; -&gt; pgbTransmissionTs)<br>    val baseTable = CUDBaseTable(&quot;&quot;, &quot;&quot;, &quot;&quot;, loadTimeStamp, &quot;&quot;)<br>    val groupBlockCUD = CUDPayload(baseTable, columnMap, &quot;&quot;, &quot;&quot;)<br>    val groupBlockPayload = GBCollationPayload(null, null, null, groupBlockCUD)<br>    val event = GroupBlockCollationEvent(null, groupBlockPayload)<br><br>    val gbList = List(event)<br><br>    val result = groupBlockHelper.filterPhMSTsList(gbList)<br><br>    assert(result.nonEmpty)<br>  }<br><br>  test(&quot;Absence: getGbEntity should return a list of GroupBlockEntity&quot;) {<br>    val PhMSTsList = List(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;A&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityId&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;789&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;PC&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;GC&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    val result = groupBlockHelper.getGbEntity(PhMSTsList)<br><br>    assert(result.nonEmpty, &quot;Result should not be empty&quot;)<br>    assert(<br>      result.head.actionIndicator.contains(&quot;A&quot;),<br>      &quot;First element&#39;s actionIndicator should be &#39;A&#39;&quot;<br>    )<br>    assert(<br>      result.head.transmissionTimestamp.contains(&quot;2022-01-01T00:00:00&quot;),<br>      &quot;First element&#39;s transmissionTimestamp should be &#39;2022-01-01T00:00:00&#39;&quot;<br>    )<br>    assert(<br>      result.head.opportunityId.contains(&quot;123&quot;),<br>      &quot;First element&#39;s opportunityId should be &#39;123&#39;&quot;<br>    )<br>    assert(<br>      result.head.quoteId.contains(&quot;456&quot;),<br>      &quot;First element&#39;s quoteId should be &#39;456&#39;&quot;<br>    )<br>    assert(<br>      result.head.confirmationNumber.contains(&quot;789&quot;),<br>      &quot;First element&#39;s confirmationNumber should be &#39;789&#39;&quot;<br>    )<br>    assert(<br>      result.head.propertyCode.contains(&quot;PC&quot;),<br>      &quot;First element&#39;s propertyCode should be &#39;PC&#39;&quot;<br>    )<br>    assert(<br>      result.head.groupCode.contains(&quot;GC&quot;),<br>      &quot;First element&#39;s groupCode should be &#39;GC&#39;&quot;<br>    )<br><br>  }<br><br>  test(&quot;Absence CMP: getGbEntity should return a list of GroupBlockEntity&quot;) {<br>    val PhMSTsList = List(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;A&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityID&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;789&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;PC&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;CMP&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    val result = groupBlockHelper.getGbEntity(PhMSTsList)<br><br>    assert(result.nonEmpty, &quot;Result should not be empty&quot;)<br>    assert(<br>      result.head.actionIndicator.contains(&quot;U&quot;),<br>      &quot;First element&#39;s actionIndicator should be &#39;U&#39;&quot;<br>    )<br>    assert(<br>      result.head.transmissionTimestamp.contains(&quot;2022-01-01T00:00:00&quot;),<br>      &quot;First element&#39;s transmissionTimestamp should be &#39;2022-01-01T00:00:00&#39;&quot;<br>    )<br>    assert(<br>      result.head.groupCode.contains(&quot;CMP&quot;),<br>      &quot;First element&#39;s groupCode should be &#39;CMP&#39;&quot;<br>    )<br><br>  }<br><br>  test(&quot;Create: getGbEntity should return a list of GroupBlockEntity&quot;) {<br>    val PhMSTsList = List(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;C&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityId&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;789&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;PC&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;GC&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    val result = groupBlockHelper.getGbEntity(PhMSTsList)<br>    println(result.toString())<br><br>    assert(result.nonEmpty, &quot;Result should not be empty&quot;)<br>    assert(<br>      result.head.actionIndicator.contains(&quot;C&quot;),<br>      &quot;First element&#39;s actionIndicator should be &#39;C&#39;&quot;<br>    )<br>    assert(<br>      result.head.transmissionTimestamp.contains(&quot;2022-01-01T00:00:00&quot;),<br>      &quot;First element&#39;s transmissionTimestamp should be &#39;2022-01-01T00:00:00&#39;&quot;<br>    )<br>    assert(<br>      result.head.opportunityId.contains(&quot;123&quot;),<br>      &quot;First element&#39;s opportunityId should be &#39;123&#39;&quot;<br>    )<br>    assert(<br>      result.head.quoteId.contains(&quot;456&quot;),<br>      &quot;First element&#39;s quoteId should be &#39;456&#39;&quot;<br>    )<br>    assert(<br>      result.head.confirmationNumber.contains(&quot;789&quot;),<br>      &quot;First element&#39;s confirmationNumber should be &#39;789&#39;&quot;<br>    )<br>    assert(<br>      result.head.propertyCode.contains(&quot;PC&quot;),<br>      &quot;First element&#39;s propertyCode should be &#39;PC&#39;&quot;<br>    )<br>    assert(<br>      result.head.groupCode.contains(&quot;GC&quot;),<br>      &quot;First element&#39;s groupCode should be &#39;GC&#39;&quot;<br>    )<br><br>  }<br><br>  test(&quot;Delete: getGbEntity should return a list of GroupBlockEntity&quot;) {<br>    val PhMSTsList = List(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;D&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityID&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;789&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;PC&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;GC&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    val result = groupBlockHelper.getGbEntity(PhMSTsList)<br><br>    assert(result.nonEmpty, &quot;Result should not be empty&quot;)<br>    assert(<br>      result.head.actionIndicator.contains(&quot;D&quot;),<br>      &quot;First element&#39;s actionIndicator should be &#39;D&#39;&quot;<br>    )<br>    assert(<br>      result.head.transmissionTimestamp.contains(&quot;2022-01-01T00:00:00&quot;),<br>      &quot;First element&#39;s transmissionTimestamp should be &#39;2022-01-01T00:00:00&#39;&quot;<br>    )<br>    assert(<br>      result.head.opportunityId.isEmpty,<br>      &quot;First element&#39;s opportunityId should be None&quot;<br>    )<br><br>  }<br><br>  test(<br>    &quot;createDeleteSeq should return a list of GroupBlockEntity with the provided fields&quot;<br>  ) {<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis()))<br>    )<br>    val startDate = new java.sql.Date(System.currentTimeMillis())<br>    val endDate =<br>      new java.sql.Date(System.currentTimeMillis() + 86400000) // one day later<br><br>    val result =<br>      groupBlockHelper.createDeleteSeq(requiredFields, startDate, endDate)<br><br>    assert(result.nonEmpty, &quot;Result should not be empty&quot;)<br>    assert(<br>      result.head.actionIndicator.contains(&quot;DS&quot;),<br>      &quot;First element&#39;s actionIndicator should be &#39;DS&#39;&quot;<br>    )<br>    assert(<br>      result.head.groupEffectiveDate.contains(startDate),<br>      &quot;First element&#39;s groupEffectiveDate should be the provided startDate&quot;<br>    )<br>    assert(<br>      result.head.groupExpireDate.contains(endDate),<br>      &quot;First element&#39;s groupExpireDate should be the provided endDate&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;createDeleteSeq should return an empty list if groupEffectiveDate equals groupExpireDate&quot;<br>  ) {<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(new java.sql.Date(System.currentTimeMillis())),<br>      Some(new java.sql.Date(System.currentTimeMillis()))<br>    )<br>    val startDate = new java.sql.Date(System.currentTimeMillis())<br>    val endDate =<br>      new java.sql.Date(System.currentTimeMillis()) // same as startDate<br><br>    val result =<br>      groupBlockHelper.createDeleteSeq(requiredFields, startDate, endDate)<br><br>    assert(result.isEmpty, &quot;Result should be empty&quot;)<br>  }<br><br>  test(<br>    &quot;getRightDates should return pgb dates if pgb_start_dt is after groupExpireDate&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-02&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-03&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;))<br>    )<br><br>    val result = groupBlockHelper.getRightDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-02&quot;),<br>      &quot;Start date should be pgb_start_dt&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-03&quot;),<br>      &quot;End date should be pgb_end_dt&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;getRightDates should return groupExpireDate and pgb_end_dt if groupExpireDate is before pgb_end_dt&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-01&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-03&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;))<br>    )<br><br>    val result = groupBlockHelper.getRightDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-02&quot;),<br>      &quot;Start date should be groupExpireDate&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-03&quot;),<br>      &quot;End date should be pgb_end_dt&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;getRightDates should return pgb_start_dt for both dates in other cases&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-01&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-01&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-01&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;))<br>    )<br><br>    val result = groupBlockHelper.getRightDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-01&quot;),<br>      &quot;Start date should be pgb_start_dt&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-01&quot;),<br>      &quot;End date should be pgb_start_dt&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;getLeftDates should return pgb dates if pgb_end_dt is before groupEffectiveDate&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-01&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-02&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-04&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-04&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-04&quot;))<br>    )<br><br>    val result = groupBlockHelper.getLeftDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-01&quot;),<br>      &quot;Start date should be pgb_start_dt&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-02&quot;),<br>      &quot;End date should be pgb_end_dt&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;getLeftDates should return pgb_start_dt and groupEffectiveDate if pgb_start_dt is before groupEffectiveDate&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-01&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-03&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;))<br>    )<br><br>    val result = groupBlockHelper.getLeftDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-01&quot;),<br>      &quot;Start date should be pgb_start_dt&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-02&quot;),<br>      &quot;End date should be groupEffectiveDate&quot;<br>    )<br>  }<br><br>  test(<br>    &quot;getLeftDates should return pgb_start_dt for both dates in other cases&quot;<br>  ) {<br>    val keyValuePair =<br>      Map(&quot;pgb_start_dt&quot; -&gt; &quot;2022-01-03&quot;, &quot;pgb_end_dt&quot; -&gt; &quot;2022-01-04&quot;)<br>    val requiredFields = GroupBlockEntity(<br>      Some(&quot;A&quot;),<br>      Some(new java.util.Date().toString),<br>      None,<br>      None,<br>      Some(&quot;123&quot;),<br>      Some(&quot;PC&quot;),<br>      Some(&quot;GC&quot;),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      None,<br>      Some(&quot;CONVENTION&quot;),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>      Some(1),<br>      Some(&quot;XXX&quot;),<br>      None,<br>      Some(1),<br>      Some(Date.valueOf(&quot;2022-01-02&quot;)),<br>      Some(Date.valueOf(&quot;2022-01-03&quot;))<br>    )<br><br>    val result = groupBlockHelper.getLeftDates(keyValuePair, requiredFields)<br><br>    assert(<br>      result._1 == Date.valueOf(&quot;2022-01-03&quot;),<br>      &quot; Start date should be pgb_start_dt&quot;<br>    )<br>    assert(<br>      result._2 == Date.valueOf(&quot;2022-01-03&quot;),<br>      &quot;End date should be pgb_start_dt&quot;<br>    )<br>  }<br><br>  test(&quot;actDF should return Seq of GroupBlockEntityNewUpdated&quot;) {<br>    val groupBlockEntity = mock[GroupBlockEntityNew]<br>    val result = groupBlockHelper.actDF(groupBlockEntity, spark)<br>    println(result.toString())<br><br>    assert(result.isInstanceOf[Seq[GroupBlockEntityNewUpdated]])<br>    assert(result.head.isInstanceOf[GroupBlockEntityNewUpdated])<br>  }<br><br>  test(<br>    &quot;actDF should return Seq of GroupBlockEntityNewUpdated when InvDateSpan &gt; 0&quot;<br>  ) {<br>    val groupBlockEntity = mock[GroupBlockEntityNew]<br>    when(groupBlockEntity.InvDateSpan).thenReturn(1)<br>    val result = groupBlockHelper.actDF(groupBlockEntity, spark)<br><br>    assert(result.isInstanceOf[Seq[GroupBlockEntityNewUpdated]])<br>    assert(result.head.isInstanceOf[GroupBlockEntityNewUpdated])<br>  }<br><br>  test(&quot;summarizeDetail should return GroupBlockCollationSchema&quot;) {<br>    val groupByKey = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      new Date(System.currentTimeMillis()),<br>      new Date(System.currentTimeMillis()),<br>      &quot;key4&quot;,<br>      &quot;key5&quot;,<br>      &quot;key6&quot;,<br>      &quot;key7&quot;,<br>      &quot;key8&quot;,<br>      &quot;key9&quot;,<br>      &quot;key10&quot;,<br>      &quot;key11&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key12&quot;,<br>      Some(1)<br>    )<br><br>    val record = mock[GroupBlockEntityUpdatedCurrencyCode]<br>    when(record.Amt).thenReturn(100.0)<br>    when(record.inventoryDate).thenReturn(new Date(System.currentTimeMillis()))<br>    when(record.ratePlanEffectiveDate).thenReturn(<br>      new Date(System.currentTimeMillis())<br>    )<br>    when(record.ratePlanExpireDate).thenReturn(<br>      new Date(System.currentTimeMillis())<br>    )<br>    when(record.blockedRooms).thenReturn(Some(10))<br>    when(record.transmissionTimestamp).thenReturn(&quot;2022-03-03T12:00:00Z&quot;)<br><br>    val records = List(record).iterator<br><br>    val result = groupBlockHelper.summarizeDetail(groupByKey, records)<br><br>    result shouldBe a[GroupBlockCollationSchema]<br>  }<br><br>  test(&quot;checkForNull should return empty string when input is null&quot;) {<br>    val result = groupBlockHelper.checkForNull(null)<br>    assert(result == &quot;&quot;)<br>  }<br><br>  test(<br>    &quot;checkForNull should return string representation of input when input is not null&quot;<br>  ) {<br>    val result = groupBlockHelper.checkForNull(&quot;test&quot;)<br>    assert(result == &quot;test&quot;)<br>  }<br><br>  test(<br>    &quot;checkForNull should return string representation of input when input is a number&quot;<br>  ) {<br>    val result = groupBlockHelper.checkForNull(123)<br>    assert(result == &quot;123&quot;)<br>  }<br><br>  test(&quot;processRevenueData should process data correctly&quot;) {<br>    // Arrange<br>    implicit val spark: SparkSession = SparkSession<br>      .builder()<br>      .appName(&quot;Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br><br>    val data = Seq(<br>      GroupBlockEntityUpdatedCurrencyCode(<br>        &quot;A&quot;,<br>        &quot;2022-03-03T12:00:00Z&quot;,<br>        &quot;A123&quot;,<br>        &quot;A123&quot;,<br>        &quot;12345&quot;,<br>        &quot;CHIRZ&quot;,<br>        &quot;AC1&quot;,<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        Date.valueOf(&quot;2022-12-31&quot;),<br>        &quot;GP1&quot;,<br>        &quot;CONVENTION&quot;,<br>        &quot;KING&quot;,<br>        &quot;ABC&quot;,<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        Date.valueOf(&quot;2022-12-31&quot;),<br>        Some(100),<br>        &quot;USD&quot;,<br>        Some(2),<br>        Some(10),<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        Date.valueOf(&quot;2022-12-31&quot;),<br>        &quot;CHIRZ&quot;,<br>        &quot;A&quot;,<br>        &quot;N&quot;,<br>        &quot;N&quot;,<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        100.0<br>      )<br>    )<br>    val ds = spark.createDataset(data)(<br>      Encoders.product[GroupBlockEntityUpdatedCurrencyCode]<br>    )<br>    val jobName = &quot;testJob&quot;<br>    val jobIdentifier = &quot;testIdentifier&quot;<br><br>    val result = groupBlockHelper.processRevenueData(ds, jobName, jobIdentifier)<br>    assert(result.count() == data.length)<br>  }<br><br><br>  test(&quot;getDbLookupBrandCode should return true when lookupData is not empty&quot;) {<br><br>    val key = &quot;CHIRZGC1789&quot;<br>    val groupBlockEvnts = Iterator.empty<br>    val DBSinkReader = new DbReader(Map.empty)<br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br><br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    when(mockDbReader.processBrandCode(anyString())).thenReturn(Option(true))<br><br>    val result = groupBlockHelper.getDbLookupBrandCode(<br>      key,<br>      mockDbReader<br>    )<br>    assert(result.isDefined)<br>  }<br><br>  test(&quot;getDbLookupBrandCode should return a boolean&quot;) {<br>    val key = &quot;CHIRZGC1789&quot;<br>    val groupBlockEvnts = Iterator.empty<br>    val query = &quot;&quot;<br>    val DBSinkReader = new DbReader(Map.empty)<br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    when(mockDbReader.processBrandCode(anyString())).thenReturn(Option(true))<br>    val result = groupBlockHelper.getDbLookupBrandCode(<br>      &quot;CHIRZGC1789&quot;,<br>      mockDbReader<br>    )<br><br>    println(&quot;resultis&quot;, result)<br>    assert(result.isInstanceOf[Some[Boolean]])<br>    // Add more assertions based on your expectations<br>  }<br><br><br>  trait ConnectionMaker {<br>    def createConnection(url: String, user: String, password: String): Connection<br>  }<br><br>  class Wrapper extends ConnectionMaker {<br>    def createConnection(url: String, user: String, password: String): Connection =<br>      DriverManager.getConnection(url, user, password)<br>  }<br>  test(&quot;Positive:getLookupDataPropParamAssignMap should return a map&quot;) {<br>    // Arrange<br>    val propCode = &quot;CHIRZ&quot;<br>    val DBSinkReader = new DbReader(Map.empty)<br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    val mockGBHelper = mock[GroupBlockHelper]<br>    val connectionMaker = mock[ConnectionMaker]<br>val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    if (resultSet.next()) {<br>      Map(&quot;ppa_prop_code&quot; -&gt; resultSet.getString(&quot;ppa_prop_code&quot;))<br>    } else {<br>      Map.empty[String, String]<br>    }<br><br>    // Act<br>    val result = groupBlockHelper.getLookupDataPropParamAssignMap(<br>      propCode = propCode,<br>      mockDbReader<br>    )<br><br>    println(&quot;Res be like&quot;, result)<br>  }<br><br>  test(<br>    &quot;Negative : getLookupDataPropParamAssignMap should return a map with null&quot;<br>  ) {<br>    val mockDbReader:DbReader = mock[DbReader]<br>    val result = groupBlockHelper.getLookupDataPropParamAssignMap(<br>      propCode = &quot;test&quot;,<br>      mockDbReader<br>        )<br><br>    println(&quot;Res be like&quot;, result)<br>    assert(result.isInstanceOf[Option[Map[String, String]]])<br>  }<br>//ankit<br>  import org.mockito.Mockito._<br><br><br>  test(&quot;Postive: getDbLookupPjGb should return a map2&quot;) {<br>    var mockedDriverManager: MockedStatic[DriverManager] = null<br><br>    try {<br>      mockedDriverManager = Mockito.mockStatic(classOf[DriverManager])<br><br>      mockedDriverManager.when(() =&gt; DriverManager.getConnection(anyString(), anyString(), anyString()))<br>        .thenReturn(connection)<br>      when(connection.createStatement).thenReturn(statement)<br>      when(statement.executeQuery(anyString())).thenReturn(resultSet)<br><br>      // ... rest of your test case ...<br><br>    } finally {<br>      if (mockedDriverManager != null) {<br>        mockedDriverManager.close()<br>      }<br>    }<br>  }<br><br><br>  test(<br>    &quot;Positive:getLookupDataPropParamAssignWithPropcodeAndParamValue should return a map&quot;<br>  ) {<br>    // Arrange<br>    val propCode = &quot;CHIRZ&quot;<br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    println(schemaMap)<br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    when(mockDbReader.processPropParam(anyString())).thenReturn(Some(&quot;ppa_param_value&quot;))<br><br>    // Act<br>    val result =<br>      groupBlockHelper.getLookupDataPropParamAssignWithPropcodeAndParamValue(<br>        propCode = propCode,<br><br>        mockDbReader<br>      )<br><br>    println(&quot;Res be like&quot;, result)<br>    assert(result === Some(&quot;ppa_param_value&quot;))<br>  }<br><br>  test(<br>    &quot;Negative : getLookupDataPropParamAssignWithPropcodeAndParamValue should return a map with null&quot;<br>  ) {<br><br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    when(mockDbReader.processPropParam(anyString())).<br>      thenReturn(Some(EmptyString))<br>    val result =<br>      groupBlockHelper.getLookupDataPropParamAssignWithPropcodeAndParamValue(<br>        propCode = &quot;test&quot;,<br>        mockDbReader<br><br>      )<br><br>    println(&quot;Res be like&quot;, result)<br>    assert(result.contains(EmptyString))<br>  }<br><br><br><br><br>  test(<br>    &quot;getInvDtLevelDataWithCurCodeDS should transform the input Dataset correctly&quot;<br>  ) {<br><br>    implicit val spark = SparkSession<br>      .builder()<br>      .appName(&quot;GroupBlockEntity Example&quot;)<br>      .master(&quot;local[*]&quot;) // Add this line if you&#39;re running Spark locally<br>      .getOrCreate()<br><br>    import spark.implicits._<br>    val data = List(<br>      GroupBlockEntityNewUpdated(<br>        actionIndicator = &quot;C&quot;,<br>        transmissionTimestamp = &quot;2022-01-01T00:00:00&quot;,<br>        opportunityId = &quot;123&quot;,<br>        quoteId = &quot;456&quot;,<br>        confirmationNumber = &quot;789&quot;,<br>        propertyCode = &quot;CHIRZ&quot;,<br>        groupCode = &quot;GA41234567&quot;,<br>        groupEffectiveDate = Date.valueOf(&quot;2022-01-01&quot;),<br>        groupExpireDate = Date.valueOf(&quot;2022-01-02&quot;),<br>        groupEventName = &quot;Test&quot;,<br>        groupEventType = &quot;CONVENTION&quot;,<br>        inventoryTypeCode = &quot;XXX&quot;,<br>        ratePlanCode = &quot;YYY&quot;,<br>        ratePlanEffectiveDate = Date.valueOf(&quot;2022-01-01&quot;),<br>        ratePlanExpireDate = Date.valueOf(&quot;2022-01-02&quot;),<br>        amount = Some(1),<br>        currencyCode = null,<br>        decimalPrecision = Some(2),<br>        blockedRooms = Some(3),<br>        inventoryEffectiveDate = Date.valueOf(&quot;2022-01-01&quot;),<br>        inventoryExpireDate = Date.valueOf(&quot;2022-01-02&quot;),<br>        ppa_prop_code = &quot;PPA1&quot;,<br>        data_source_ind = &quot;DSI&quot;,<br>        groupIndicator = &quot;GI&quot;,<br>        groupSourceIndicator = &quot;GSI&quot;,<br>        inventoryDate = Date.valueOf(&quot;2022-01-01&quot;),<br>        Amt = 100.0<br>      )<br>    )<br>    val groupBlockDataset = data.toDS()<br><br>    val mockedDbLookup = mock[GroupBlockHelper]<br><br>    val schemaMap = jobConfigDetails.schemamap.get<br><br>    groupBlockDataset.show(false)<br><br>    val mockDbReader:DbReader = mock[DbReader]<br><br>    when(mockDbReader.open()).thenReturn(true)<br>    when(<br>      mockedDbLookup.getLookupDataPropParamAssignWithPropcodeAndParamValue(<br>        anyString(),<br>        //any()<br>        any[DbReader]()<br>      )(ArgumentMatchers.any(classOf[ConnectionConfigDetails]),ArgumentMatchers.any(classOf[JobConfigurationDetails]),ArgumentMatchers.any())<br>    ).thenReturn(Option(&quot;USD&quot;))<br>    val result = groupBlockHelper.getInvDtLevelDataWithCurCodeDS(<br>      groupBlockDataset,<br>      schemaMap<br>    )<br><br>    assert(result.head().propertyCode != null)<br>  }<br><br><br><br><br><br>  test(<br>    &quot;summarizeDetail should return GroupBlockCollationSchema when there are zeroAmtRecord&quot;<br>  ) {<br>    val groupByKey = (<br>      &quot;key1&quot;,<br>      &quot;key2&quot;,<br>      &quot;key3&quot;,<br>      new Date(System.currentTimeMillis()),<br>      new Date(System.currentTimeMillis()),<br>      &quot;key4&quot;,<br>      &quot;key5&quot;,<br>      &quot;key6&quot;,<br>      &quot;key7&quot;,<br>      &quot;key8&quot;,<br>      &quot;key9&quot;,<br>      &quot;key10&quot;,<br>      &quot;key11&quot;,<br>      new Date(System.currentTimeMillis()),<br>      &quot;key12&quot;,<br>      Some(1)<br>    )<br><br>    val record = mock[GroupBlockEntityUpdatedCurrencyCode]<br>    when(record.Amt).thenReturn(0.0)<br>    when(record.inventoryDate).thenReturn(new Date(System.currentTimeMillis()))<br>    when(record.ratePlanEffectiveDate).thenReturn(<br>      new Date(System.currentTimeMillis())<br>    )<br>    when(record.ratePlanExpireDate).thenReturn(<br>      new Date(System.currentTimeMillis())<br>    )<br>    when(record.blockedRooms).thenReturn(Some(10))<br>    when(record.transmissionTimestamp).thenReturn(&quot;2022-03-03T12:00:00Z&quot;)<br><br>    val records = List(record).iterator<br><br>    val result = groupBlockHelper.summarizeDetail(groupByKey, records)<br><br>    result shouldBe a[GroupBlockCollationSchema]<br><br>    result.inventoryDate shouldBe record.inventoryDate<br>    result.blockedRooms shouldBe record.blockedRooms.getOrElse(0)<br>    result.numRecs shouldBe 1<br>    result.avgAmt shouldBe 0.0<br>    result.blockedRevenue shouldBe 0.0<br>  }<br><br>  test(<br>    &quot;getLookupDataPropParamAssignMap should handle when the resultset is empty&quot;<br>  ) {<br><br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(mockDbReader.open()).thenReturn(true)<br>    when(mockDbReader.processBrandCode(anyString())).thenReturn(Option(true))<br><br>    val result = groupBlockHelper.getLookupDataPropParamAssignMap(<br>      &quot;CHIRZGA4123456712345&quot;,<br><br>      mockDbReader<br>    )<br><br>    assert(result.isInstanceOf[Option[Map[String, String]]])<br>  }<br><br>  test(<br>    &quot;getLookupDataPropParamAssignWithPropcodeAndParamValue should return an empty string&quot;<br>  ) {<br><br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    val mockDbReader:DbReader = mock[DbReader]<br>    when(connection.createStatement).thenReturn(statement)<br>    when(statement.executeQuery(anyString())).thenReturn(resultSet)<br>    when(resultSet.next()).thenReturn(false)<br><br>    val result =<br>      groupBlockHelper.getLookupDataPropParamAssignWithPropcodeAndParamValue(<br>        &quot;CHIRZGA4123456712345&quot;,<br><br>        mockDbReader<br>      )<br><br>    assert(result.isInstanceOf[Option[String]])<br>  }<br>  var pst:PreparedStatement = _<br><br>  test(&quot;calculateBlockedRevenue should transform the input Dataset correctly&quot;) {<br>    // Arrange<br>    implicit val spark = SparkSession.builder()<br>      .appName(&quot;Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br><br>    val gbCollationEvent = Seq(<br>      GroupBlockCollationEvent(<br>        groupBlockInfo = null, // Set to null<br>        groupBlockPayload = GBCollationPayload(<br>          payLoad = &quot;samplePayload&quot;,<br>          payLoadType = &quot;JSON&quot;,<br>          groupBlockCollation = None, // Set to None for Option type<br>          groupBlockCUD = CUDPayload(<br>            baseTable = CUDBaseTable(<br>              table = &quot;sampleTable&quot;,<br>              //              rowId = &quot;rows_1234&quot;,<br>              rowId = &quot;TULTDACBD45886335&quot;,<br>              modNum = &quot;sampleModNum&quot;,<br>              loadTimeStamp = &quot;sampleTimeStamp&quot;,<br>              verb = &quot;sampleVerb&quot;<br>            ),<br>            columnMap = Map(&quot;sampleKey&quot; -&gt; &quot;sampleValue&quot;),<br>            payLoad = &quot;samplePayload&quot;,<br>            payLoadType = &quot;samplePayloadType&quot;,<br>            isPresentInDb = true<br>          )<br>        )<br>      )<br>    )<br>    val gbDataset = spark.createDataset(gbCollationEvent)(Encoders.product[GroupBlockCollationEvent])<br><br>    // Create mock instances of the implicit parameters<br>    implicit val dbDetails: DbDetails = mock[DbDetails]<br>    implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>    implicit val jobConfig: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>    implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;, &quot;&quot;, &quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>      &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br><br><br>    // Act<br>    val result = groupBlockHelper.calculateBlockedRevenue(gbDataset)<br><br>    // Assert<br>    assert(result.count() == gbDataset.count())<br>  }<br><br><br>  //revision_4<br>  test(&quot;transformGroupBlockRecords transforms records correctly&quot;) {<br><br>    val loadTimeStamp = &quot;2022-12-21T23:59:50Z&quot;<br>    val baseTable = CUDBaseTable(&quot;&quot;, &quot;&quot;, &quot;&quot;, loadTimeStamp, &quot;&quot;)<br>    val columnMap = Map(<br>      &quot;pgb_transmission_ts&quot; -&gt; &quot;2022-03-15 10:15:31.123&quot;,<br>      &quot;verb&quot; -&gt; &quot;A&quot;,<br>      &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>      &quot;opportunityID&quot; -&gt; &quot;123&quot;,<br>      &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>      &quot;confirmationNumber&quot; -&gt; &quot;789&quot;,<br>      &quot;propertyCode&quot; -&gt; &quot;PC&quot;,<br>      &quot;groupCode&quot; -&gt; &quot;CMP&quot;,<br>      &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>      &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>      &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>      &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>      &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>      &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>      &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>      &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>      &quot;amount&quot; -&gt; &quot;1&quot;,<br>      &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>      &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>      &quot;blocked&quot; -&gt; &quot;3&quot;,<br>      &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>      &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>    )<br>    val groupBlockCUD = CUDPayload(baseTable, columnMap, &quot;&quot;, &quot;&quot;)<br>    val groupBlockPayload = GBCollationPayload(null, null, null, groupBlockCUD)<br>    val event = GroupBlockCollationEvent(null, groupBlockPayload)<br>    val iterator = Iterator(event)<br><br><br>    val result =<br>      groupBlockHelper.transformGroupBlockRecords(&quot;CHIRZGA4123456712345&quot;, iterator)<br><br>    assert(result.length == 1)<br>  }<br>  test(&quot;getDbLookupPjGb should handle when the resultset is empty&quot;) {<br>    val schemaMap: Map[String, String] = jobConfigDetails.schemamap.get<br>    val DBSinkReader = new DbReader(Map.empty)<br>    val mockDbReader: DbReader = mock[DbReader]<br>    val gbCollation = Iterator.empty<br>    when(resultSet.next()).thenReturn(false)<br>    val result = groupBlockHelper.getDbLookupPjGb(<br>      &quot;CHIRZGA4123456712345&quot;,<br>      mockDbReader<br>    )<br><br>    assert(result.isInstanceOf[Option[Map[String, String]]])<br>  }<br><br><br>  test(&quot;processGbEntity should transform the input correctly for create operation&quot;) {<br>    // Arrange<br>    implicit val spark = SparkSession.builder()<br>      .appName(&quot;Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br>    import spark.implicits._<br><br>    val data = List(<br>      GroupBlockEntity(<br>        Some(&quot;A&quot;),<br>        Some(new java.util.Date().toString),<br>        None,<br>        None,<br>        Some(&quot;ABCD0001&quot;), //123<br>        Some(&quot;TULTD&quot;),<br>        Some(&quot;GA41234567&quot;), //GC<br>        Some(new java.sql.Date(System.currentTimeMillis())),<br>        Some(new java.sql.Date(System.currentTimeMillis())),<br>        None,<br>        Some(&quot;CONVENTION&quot;),<br>        Some(&quot;XXX&quot;),<br>        None,<br>        Some(new java.sql.Date(System.currentTimeMillis())),<br>        Some(new java.sql.Date(System.currentTimeMillis())),<br>        Some(1),<br>        Some(&quot;XXX&quot;),<br>        None,<br>        Some(1),<br>        Some(new java.sql.Date(System.currentTimeMillis())),<br>        Some(new java.sql.Date(System.currentTimeMillis()))<br>      )<br>    )<br><br>    val iterator = Iterator(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;A&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityId&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;ABCD0001&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;TULTD&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;GA41234567&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    implicit val dbDetails: DbDetails = mock[DbDetails]<br>    implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>    implicit val jobConfig: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>    implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;, &quot;&quot;, &quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>      &quot;source&quot;, &quot;target&quot;, &quot;status&quot;, &quot;traceId&quot;, &quot;spanId&quot;)<br>    val maps = Map(<br>      &quot;keyValueSchema&quot; -&gt; Map(&quot;pgb_transmission_ts&quot; -&gt; &quot;2022-01-01 00:00:00&quot;),<br>      &quot;schemaMap&quot; -&gt; Map(&quot;someKey&quot; -&gt; &quot;someValue&quot;)<br>    )<br><br><br>    // Act<br>    val result = groupBlockHelper.processGbEntity(maps, data, &quot;TULTDACBD45886335&quot;, iterator)<br><br>    // Assert<br>    assert(result.head.propertyCode != null)<br><br>  }<br>  test(&quot;processGbEntity should transform the input correctly for empty operation&quot;) {<br>    // Arrange<br>    implicit val spark = SparkSession.builder()<br>      .appName(&quot;Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br>    import spark.implicits._<br><br>    val data = List(<br>      GroupBlockEntity(<br>        Some(&quot;H&quot;),<br>        Some(new java.util.Date().toString),<br>        None,<br>        None,<br>        Some(&quot;ABCD0001&quot;), //123<br>        Some(&quot;TULTD&quot;),<br>        Some(&quot;GA41234567&quot;), //GC<br>        Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>        Some(Date.valueOf(&quot;2022-01-04&quot;)),<br>        None,<br>        Some(&quot;CONVENTION&quot;),<br>        Some(&quot;XXX&quot;),<br>        None,<br>        Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>        Some(Date.valueOf(&quot;2022-01-04&quot;)),<br>        Some(1),<br>        Some(&quot;XXX&quot;),<br>        None,<br>        Some(1),<br>        Some(Date.valueOf(&quot;2022-01-03&quot;)),<br>        Some(Date.valueOf(&quot;2022-01-04&quot;)),<br>      )<br>    )<br><br>    val iterator = Iterator(<br>      GroupBlockCollationEvent(<br>        null,<br>        GBCollationPayload(<br>          null,<br>          null,<br>          null,<br>          CUDPayload(<br>            null,<br>            columnMap = Map(<br>              &quot;verb&quot; -&gt; &quot;H&quot;,<br>              &quot;generationTime&quot; -&gt; &quot;2022-01-01T00:00:00&quot;,<br>              &quot;opportunityId&quot; -&gt; &quot;123&quot;,<br>              &quot;quoteId&quot; -&gt; &quot;456&quot;,<br>              &quot;confirmationNumber&quot; -&gt; &quot;ABCD0001&quot;,<br>              &quot;propertyCode&quot; -&gt; &quot;TULTD&quot;,<br>              &quot;groupCode&quot; -&gt; &quot;GA41234567&quot;,<br>              &quot;groupEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;groupExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;groupName&quot; -&gt; &quot;Test&quot;,<br>              &quot;groupType&quot; -&gt; &quot;CONVENTION&quot;,<br>              &quot;inventoryTypeCode&quot; -&gt; &quot;XXX&quot;,<br>              &quot;ratePlanCode&quot; -&gt; &quot;YYY&quot;,<br>              &quot;ratePlanEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;ratePlanExpireDate&quot; -&gt; &quot;2022-01-02&quot;,<br>              &quot;amount&quot; -&gt; &quot;1&quot;,<br>              &quot;currencyCode&quot; -&gt; &quot;USD&quot;,<br>              &quot;dec_pre&quot; -&gt; &quot;2&quot;,<br>              &quot;blocked&quot; -&gt; &quot;3&quot;,<br>              &quot;inventoryEffectiveDate&quot; -&gt; &quot;2022-01-01&quot;,<br>              &quot;inventoryExpireDate&quot; -&gt; &quot;2022-01-02&quot;<br>            ),<br>            &quot;&quot;,<br>            &quot;&quot;<br>          )<br>        )<br>      )<br>    )<br><br>    implicit val dbDetails: DbDetails = mock[DbDetails]<br>    implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>    implicit val jobConfig: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>    implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;, &quot;&quot;, &quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>      &quot;source&quot;, &quot;target&quot;, &quot;status&quot;, &quot;traceId&quot;, &quot;spanId&quot;)<br>    val schemaMap: Map[String, String] =<br>      jobConfigDetails.schemamap.get ++ connConfigDetails.schemaconfig.get<br><br>    val maps = Map(<br>      &quot;keyValueSchema&quot; -&gt; Map(&quot;pgb_transmission_ts&quot; -&gt; &quot;2022-01-01 00:00:00&quot;,&quot;pgb_end_dt&quot; -&gt;&quot;2022-01-04&quot;,&quot;pgb_start_dt&quot; -&gt;&quot;2022-01-02&quot;),<br>      &quot;schemaMap&quot; -&gt; Map(&quot;someKey&quot; -&gt; &quot;someValue&quot;)<br>    )<br>    // Act<br>    val result = groupBlockHelper.processGbEntity(maps, data, &quot;TULTDACBD45886335&quot;, iterator) //Map.empty)<br><br>    // Assert<br>    assert(result.head.propertyCode != null)<br>  }<br>}<br>jobs<br>collation job:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.jobs<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.entity.CustomLoggerInfo<br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import com.marriott.mgpapp.spark.groupblock.collation.entity.ReservationGroupBlockMerge<br>import com.marriott.mgpapp.spark.groupblock.collation.jobs.GroupBlockCollationJob<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper<br>import com.marriott.mgpapp.spark.groupblock.collation.test.TestsHelper.genarateEEHData<br>import com.marriott.rm.platform.model.ProducerOutput<br>import com.marriott.rm.platform.util.{ConsumerHelper, ProducerHelper}<br>import org.apache.avro.SchemaParseException<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.apache.kafka.common.errors.TopicAuthorizationException<br>import org.apache.spark.sql.SparkSession<br>import org.mockito.Mockito.when<br>import org.mockito.MockitoSugar.mock<br>import org.scalatest._<br>import org.scalatest.funsuite.AnyFunSuite<br><br>class GroupBlockCollationJobTest extends AnyFunSuite with BeforeAndAfterAll{<br><br>  UserGroupInformation.setLoginUser(<br>    UserGroupInformation.createRemoteUser(&quot;hduser&quot;)<br>  )<br><br>  implicit var spark: SparkSession = _<br>  implicit val jobConfigDetails: JobConfigurationDetails = TestsHelper.jobConfigDetails<br>  implicit val connConfigDetails: ConnectionConfigDetails = TestsHelper.connConfigDetails<br>  implicit val dbDetails: DbDetails = TestsHelper.dbDetails<br>  var job: GroupBlockCollationJob = _<br>  implicit val logObj: CustomLoggerInfo = CustomLoggerInfo(&quot;&quot;,&quot;&quot;,&quot;appNameForLogger&quot;, &quot;project&quot;, &quot;flow&quot;,<br>    &quot;source&quot;, &quot;target&quot;, &quot;status&quot;,&quot;traceId&quot;, &quot;spanId&quot;)<br><br>  override def beforeAll(): Unit = {<br>    spark = SparkSession.builder().appName(groupBlockCollationJobTest).master(local).getOrCreate()<br>    job = new GroupBlockCollationJob()<br>  }<br><br><br>  test(&quot;consumeDataFromEEH should return DataFrame&quot;) {<br><br>    val data = genarateEEHData(spark)<br>    val mockConsumerHelper = mock[ConsumerHelper[Nothing]]<br><br>    when(mockConsumerHelper.consume()).thenReturn(data)<br>    assert(!job.consumeDataFromEEH(mockConsumerHelper).isEmpty)<br><br>    when(mockConsumerHelper.consume()).thenThrow(new SchemaParseException(&quot;Schema Pase Exception&quot;))<br>    assert(job.consumeDataFromEEH(mockConsumerHelper).isEmpty)<br>  }<br><br><br>  test(&quot;produceDataToEEH should write data to Kafka&quot;) {<br>    val mockProducer = mock[ProducerHelper[ReservationGroupBlockMerge]]<br><br>    val producerOutput = ProducerOutput(Option(true),None,None)<br>    when(mockProducer.produce()).thenReturn(Option(producerOutput))<br><br>    val result = job.produceDataToEEH(mockProducer)<br>    assert(result.isDefined)<br>    assert(result.get.isSavedToKafka.get)<br><br>    when(mockProducer.produce()).thenThrow(new TopicAuthorizationException(&quot;Topic Invalid&quot;))<br>    assert(job.produceDataToEEH(mockProducer).isEmpty)<br>  }<br><br><br><br><br>}<br>udfs<br><br>sparkUdftest:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test.udfs<br><br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import com.marriott.mgpapp.spark.groupblock.collation.udfs.SparkUdfs._<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.apache.spark.sql.SparkSession<br>import org.apache.spark.sql.functions.col<br>import org.apache.spark.sql.types._<br>import org.scalatest.funsuite.AnyFunSuite<br><br>import java.sql.Date<br><br>class SparkUdfsTest extends AnyFunSuite {<br>  UserGroupInformation.setLoginUser(<br>    UserGroupInformation.createRemoteUser(&quot;hduser&quot;)<br>  )<br>  // Create a SparkSession<br>  implicit val spark =<br>    SparkSession.builder.appName(sparkUdfsTest).master(local).getOrCreate()<br><br>  // Import the implicit SparkSession<br><br>  import spark.implicits._<br><br>  test(&quot;EmptyArray UDF should return an empty array&quot;) {<br><br>    // Create a DataFrame<br>    val df = Seq(testData).toDF(testColumn)<br><br>    // Apply the UDF<br>    val resultDF = df.withColumn(emptyColumn, EmptyArray())<br><br>    val expectedDF =<br>      Seq((testData, Array.empty[Int])).toDF(testColumn, emptyColumn)<br>    val expectedSchema = StructType(<br>      Seq(<br>        StructField(testColumn, StringType, nullable = true),<br>        StructField(<br>          emptyColumn,<br>          ArrayType(IntegerType, containsNull = false),<br>          nullable = true<br>        )<br>      )<br>    )<br><br>    // Collect the result<br>    val result = resultDF.select(emptyColumn).as[Array[Int]].collect()<br><br>    // Check that the result is an empty array<br>    assert(result(0).isEmpty)<br>    assert(resultDF.count() == 1)<br>    assert(resultDF.columns.length == 2)<br>    assert(resultDF.columns.contains(testColumn))<br>    assert(resultDF.columns.contains(emptyColumn))<br>    assert(resultDF.schema == expectedSchema)<br>    assert(resultDF.collect() sameElements expectedDF.collect())<br>  }<br><br>  test(<br>    &quot;EmptyArray UDF should return an empty array for each row in a DataFrame&quot;<br>  ) {<br>    val df = Seq(testData, testData, testData).toDF(testColumn)<br>    val resultDF = df.withColumn(emptyColumn, EmptyArray())<br>    val result = resultDF.select(emptyColumn).as[Array[Int]].collect()<br>    assert(result.forall(_.isEmpty))<br>  }<br><br>  test(&quot;EmptyArray UDF should work correctly with columns of different types&quot;) {<br>    val df = Seq((testData, 1), (testData, 2), (testData, 3))<br>      .toDF(testColumn, anotherColumn)<br>    val resultDF = df.withColumn(emptyColumn, EmptyArray())<br>    val result = resultDF.select(emptyColumn).as[Array[Int]].collect()<br>    assert(result.forall(_.isEmpty))<br>  }<br><br>  test(&quot;EmptyArray UDF should be callable by its name when registered&quot;) {<br>    spark.udf.register(emptyArray, EmptyArray)<br>    val df = Seq(testData).toDF(testColumn)<br>    val resultDF = df.selectExpr(s&quot;$testColumn&quot;, &quot;EmptyArray() as emptyArray&quot;)<br>    val result = resultDF.select(emptyArray).as[Array[Int]].collect()<br>    assert(result(0).isEmpty)<br>  }<br><br>  test(&quot;getAmount UDF should return the correct amount&quot;) {<br>    // Create a DataFrame<br>    val df = Seq((100, 2), (200, 5), (300, 3), (400, 0))<br>      .toDF(amountColumn, precisionColumn)<br><br>    // Apply the UDF<br>    val resultDF = df.withColumn(<br>      resultColumn,<br>      getAmount(col(amountColumn), col(precisionColumn))<br>    )<br><br>    val expectedSchema = StructType(<br>      Seq(<br>        StructField(amountColumn, IntegerType, nullable = false),<br>        StructField(precisionColumn, IntegerType, nullable = false),<br>        StructField(resultColumn, DoubleType, nullable = false)<br>      )<br>    )<br>    val expectedDF = Seq(<br>      (100, 2, 1.0),<br>      (200, 5, 200.0),<br>      (300, 3, 0.3),<br>      (400, 0, 400.0)<br>    ).toDF(amountColumn, precisionColumn, resultColumn)<br><br>    assert(resultDF.schema == expectedSchema)<br>    assert(resultDF.count() == 4)<br>    assert(resultDF.columns.length == 3)<br>    assert(resultDF.columns.contains(amountColumn))<br>    assert(resultDF.columns.contains(precisionColumn))<br>    assert(resultDF.columns.contains(resultColumn))<br>    assert(resultDF.collect() sameElements expectedDF.collect())<br>  }<br><br>  test(&quot;getAmount UDF should work correctly with negative amount&quot;) {<br>    val df = Seq((-100, 2)).toDF(amountColumn, precisionColumn)<br>    val resultDF = df.withColumn(<br>      resultColumn,<br>      getAmount(col(amountColumn), col(precisionColumn))<br>    )<br>    val result = resultDF.select(resultColumn).as[Double].collect()<br>    assert(result(0) == -1.0) // -100 / 10^2<br>  }<br><br>  test(&quot;getAmount UDF should work correctly with zero amount&quot;) {<br>    val df = Seq((0, 2)).toDF(amountColumn, precisionColumn)<br>    val resultDF = df.withColumn(<br>      resultColumn,<br>      getAmount(col(amountColumn), col(precisionColumn))<br>    )<br>    val result = resultDF.select(resultColumn).as[Double].collect()<br>    assert(result(0) == 0.0) // 0 / 10^2<br>  }<br><br>  test(&quot;getAmount UDF should work correctly with negative precision&quot;) {<br>    val df = Seq((100, -2)).toDF(amountColumn, precisionColumn)<br>    val resultDF = df.withColumn(<br>      resultColumn,<br>      getAmount(col(amountColumn), col(precisionColumn))<br>    )<br>    val result = resultDF.select(resultColumn).as[Double].collect()<br>    assert(result(0) == 100.0) // dec_prec not in 1 to 4<br>  }<br><br>  test(&quot;getAmount UDF should work correctly with zero precision&quot;) {<br>    val df = Seq((100, 0)).toDF(amountColumn, precisionColumn)<br>    val resultDF = df.withColumn(<br>      resultColumn,<br>      getAmount(col(amountColumn), col(precisionColumn))<br>    )<br>    val result = resultDF.select(resultColumn).as[Double].collect()<br>    assert(result(0) == 100.0) // dec_prec not in 1 to 4<br>  }<br><br>  test(&quot;getInventoryDateList UDF should return the correct date list&quot;) {<br>    // Create a DataFrame<br>    val df = Seq(<br>      (&quot;D&quot;, Date.valueOf(&quot;2022-01-01&quot;), 0),<br>      (&quot;X&quot;, Date.valueOf(&quot;2022-01-01&quot;), 3)<br>    ).toDF(verbColumn, invStartDtColumn, invDtSpanColumn)<br><br>    // Apply the UDF<br>    val resultDF = df.withColumn(<br>      invDtListColumn,<br>      getInventoryDateList(<br>        col(verbColumn),<br>        col(invStartDtColumn),<br>        col(invDtSpanColumn)<br>      )<br>    )<br>    val expectedSchema = StructType(<br>      Seq(<br>        StructField(verbColumn, StringType, nullable = true),<br>        StructField(invStartDtColumn, DateType, nullable = true),<br>        StructField(invDtSpanColumn, IntegerType, nullable = false),<br>        StructField(<br>          invDtListColumn,<br>          ArrayType(DateType, containsNull = true),<br>          nullable = true<br>        )<br>      )<br>    )<br>    val expectedDF = Seq(<br>      (&quot;D&quot;, Date.valueOf(&quot;2022-01-01&quot;), 0, Seq(Date.valueOf(&quot;9999-12-31&quot;))),<br>      (<br>        &quot;X&quot;,<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        3,<br>        Seq(<br>          Date.valueOf(&quot;2022-01-01&quot;),<br>          Date.valueOf(&quot;2022-01-02&quot;),<br>          Date.valueOf(&quot;2022-01-03&quot;)<br>        )<br>      )<br>    ).toDF(verbColumn, invStartDtColumn, invDtSpanColumn, invDtListColumn)<br><br>    assert(resultDF.schema == expectedSchema)<br>    assert(resultDF.collect() sameElements expectedDF.collect())<br>    assert(resultDF.count() == 2)<br>    assert(resultDF.columns.length == 4)<br>    assert(resultDF.columns.contains(verbColumn))<br>    assert(resultDF.columns.contains(invStartDtColumn))<br>    assert(resultDF.columns.contains(invDtSpanColumn))<br>    assert(resultDF.columns.contains(invDtListColumn))<br>  }<br><br>  test(&quot;getInventoryDateList UDF should work correctly with empty verb&quot;) {<br>    val df = Seq((&quot;&quot;, Date.valueOf(&quot;2022-01-01&quot;), 3))<br>      .toDF(verbColumn, invStartDtColumn, invDtSpanColumn)<br>    val resultDF = df.withColumn(<br>      invDtListColumn,<br>      getInventoryDateList(<br>        col(verbColumn),<br>        col(invStartDtColumn),<br>        col(invDtSpanColumn)<br>      )<br>    )<br>    val result = resultDF.select(invDtListColumn).as[Seq[Date]].collect()<br>    assert(<br>      result(0) == Seq(<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        Date.valueOf(&quot;2022-01-02&quot;),<br>        Date.valueOf(&quot;2022-01-03&quot;)<br>      )<br>    ) // verb != &quot;D&quot; and invDtSpan == 3<br>  }<br><br>  test(&quot;getInventoryDateList UDF should work correctly with zero date span&quot;) {<br>    val df = Seq((&quot;X&quot;, Date.valueOf(&quot;2022-01-01&quot;), 0))<br>      .toDF(verbColumn, invStartDtColumn, invDtSpanColumn)<br>    val resultDF = df.withColumn(<br>      invDtListColumn,<br>      getInventoryDateList(<br>        col(verbColumn),<br>        col(invStartDtColumn),<br>        col(invDtSpanColumn)<br>      )<br>    )<br>    val result = resultDF.select(invDtListColumn).as[Seq[Date]].collect()<br>    assert(result(0).isEmpty) // invDtSpan == 0<br>  }<br><br>  test(<br>    &quot;getInventoryDateList UDF should work correctly with negative date span&quot;<br>  ) {<br>    val df = Seq((&quot;X&quot;, Date.valueOf(&quot;2022-01-01&quot;), -3))<br>      .toDF(verbColumn, invStartDtColumn, invDtSpanColumn)<br>    val resultDF = df.withColumn(<br>      invDtListColumn,<br>      getInventoryDateList(<br>        col(verbColumn),<br>        col(invStartDtColumn),<br>        col(invDtSpanColumn)<br>      )<br>    )<br>    val result = resultDF.select(invDtListColumn).as[Seq[Date]].collect()<br>    assert(result(0).isEmpty) // invDtSpan &lt; 0<br>  }<br><br>  test(&quot;getInventoryDateList UDF should work correctly with null verb&quot;) {<br>    val df = Seq((null, Date.valueOf(&quot;2022-01-01&quot;), 3))<br>      .toDF(verbColumn, invStartDtColumn, invDtSpanColumn)<br>    val resultDF = df.withColumn(<br>      invDtListColumn,<br>      getInventoryDateList(<br>        col(verbColumn),<br>        col(invStartDtColumn),<br>        col(invDtSpanColumn)<br>      )<br>    )<br>    val result = resultDF.select(invDtListColumn).as[Seq[Date]].collect()<br>    assert(<br>      result(0) == Seq(<br>        Date.valueOf(&quot;2022-01-01&quot;),<br>        Date.valueOf(&quot;2022-01-02&quot;),<br>        Date.valueOf(&quot;2022-01-03&quot;)<br>      )<br>    ) // verb != &quot;D&quot; and invDtSpan == 3<br>  }<br><br>}<br><br>testhelper:<br><br>package com.marriott.mgpapp.spark.groupblock.collation.test<br><br>import com.github.javafaker.Faker<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, DbDetails, JobConfigurationDetails}<br>import com.marriott.mgpapp.spark.groupblock.Constants.IrrecoverableErrorTopic<br>import com.marriott.mgpapp.spark.groupblock.collation.TestConstants._<br>import org.apache.spark.sql.{DataFrame, SparkSession}<br><br><br>object TestsHelper {<br>  private lazy val faker = new Faker()<br><br>  def generateListOfString() = {<br>    faker.lorem().sentences(10)<br>  }<br><br>  def generateString = {<br>    faker.lorem().fixedString(5000)<br>  }<br><br>  def randomLong(): Long =<br>    faker.number().randomNumber()<br><br>  // Function to generate a random sequence of integers of a given length<br>  def randomIntSeq(length: Int): Seq[Int] =<br>    Seq.fill(length)(faker.number().randomDigit())<br><br>  // Function to generate a random sequence of booleans of a given length<br>  def randomBoolSeq(length: Int): Seq[Boolean] =<br>    Seq.fill(length)(faker.bool().bool())<br><br>  def genarateEEHData(sparkSession: SparkSession): DataFrame = {<br>    import sparkSession.implicits._<br>    sparkSession.createDataset(List(TestsHelper.toString)).as(&quot;value&quot;).toDF()<br>  }<br><br>  val jobConfigDetails: JobConfigurationDetails = JobConfigurationDetails(jobname = jobName,<br>    jobidentifier = jobIdentifier, triggerinterval = Some(100),<br>    checkpointdirectory = Some(Map(directory -&gt; testDirectory,<br>      GroupBlockCollationIrrecoverableEntityCheckpointDir -&gt; GroupBlockCollationIrrecoverableEntityCheckpointDir)),<br>  schemamap = Some(Map(&quot;TGTGroupBlockTable&quot; -&gt; &quot;TGTGroupBlockTable&quot;,<br>      &quot;PropertyTableIN&quot; -&gt; &quot;PropertyTableIN&quot;,<br>      &quot;UserControlTableIN&quot; -&gt; &quot;UserControlTableIN&quot;,<br>      &quot;PropParamAssignTableIN&quot; -&gt; &quot;PropParamAssignTableIN&quot;,<br>    &quot;SchemaName&quot;-&gt;&quot;test&quot;)),<br>    kafkatopicmap = Some(Map(IrrecoverableErrorTopic -&gt; IrrecoverableErrorTopic,<br>      IrrecoverableFilePath -&gt; IrrecoverableFilePath)))<br><br><br><br><br>  val dbDetails = DbDetails(jdbcurl = &quot;jdbc:postgresql://abc.com:5432/&quot;, drivername = &quot;org.postgresql.Driver&quot;, username = &quot;abcde&quot;,<br>    password = &quot;password&quot;, poolsize = poolsize, idletimeout = 1L, querytimeout = null,<br>    jdbcurlro = dbUrl, drivernamero = dbDrivername, usernamero = dbUsername, passwordro = dbPassword,<br>    poolsizero = poolsize, idletimeoutro = 1L, querytimeoutro = null,<br>    jdbcurlwo = dbUrl, drivernamewo = dbDrivername, usernamewo = dbUsername, passwordwo = dbPassword,<br>    poolsizewo = poolsize, idletimeoutwo = 1L, querytimeoutwo = null)<br><br><br>  val connConfigDetails: ConnectionConfigDetails = ConnectionConfigDetails(dbconfig = Some(dbDetails),<br>    bucketconfig = Some(Map(default -&gt; testBucket)),<br>    schemaconfig = Some(Map(schemaName -&gt; testSchema)))<br>}<br><br><br><br><br><br><br><br><br></pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=412f129e92fd" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[Sap Backup-2]]></title>
            <link>https://medium.com/@ankitjuneja/sap-backup-2-5eadb665d1e0?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/5eadb665d1e0</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Fri, 22 Aug 2025 04:01:16 GMT</pubDate>
            <atom:updated>2025-08-22T04:01:16.766Z</atom:updated>
            <content:encoded><![CDATA[<p>#token use before</p><pre>our main motive was to remove the token but in the above approach it is using the tokens <br><br>We want to prioritize the use of IAM roles for connecting to Databricks. IAM roles are considered a more secure and managed approach compared to tokens, especially in AWS environments. <br><br><br><br>this is our code used with token , just replace the token with Iam approach <br><br>import json<br>import requests<br>import time<br><br>def lambda_handler(event, context):<br>    # Environment setup<br>    url = &#39;https://dbc-d1e6f9d8-1cd0.cloud.databricks.com&#39;<br>    token = &#39;dapi01aab98a931788f4ca4935b106515e26&#39;<br>    warehouse_id = &quot;7a2c590f900bb013&quot;<br>    query = &quot;SELECT * FROM prod.triton_individual LIMIT 2&quot;<br><br>    # Function to perform a POST request<br>    def post_request(url, token, endpoint, json_params):<br>        full_url = f&#39;{url}/api/2.0/{endpoint}&#39;<br>        headers = {&#39;Authorization&#39;: f&#39;Bearer {token}&#39;}<br>        response = requests.post(full_url, headers=headers, json=json_params)<br>        response.raise_for_status()<br>        return response.json()<br><br>    # Function to perform a GET request<br>    def get_request(url, token, endpoint):<br>        full_url = f&#39;{url}/api/2.0/{endpoint}&#39;<br>        headers = {&#39;Authorization&#39;: f&#39;Bearer {token}&#39;}<br>        response = requests.get(full_url, headers=headers)<br>        response.raise_for_status()<br>        return response.json()<br><br>    # Function to run the SQL query<br>    def run_sql_query(url, token, query, warehouse_id):<br>        sql_endpoint = &quot;sql/statements&quot;<br>        response = post_request(url, token, sql_endpoint, {&quot;statement&quot;: query, &quot;warehouse_id&quot;: warehouse_id})<br>        <br>        if &quot;statement_id&quot; in response:<br>            statement_id = response[&quot;statement_id&quot;]<br>            # Wait for the query to execute<br>            time.sleep(5)<br>            status_endpoint = f&quot;sql/statements/{statement_id}&quot;<br>            status_response = get_request(url, token, status_endpoint)<br><br>            # Check the query status<br>            if status_response.get(&quot;status&quot;, {}).get(&quot;state&quot;) == &quot;SUCCEEDED&quot;:<br>                return status_response.get(&quot;result&quot;, {}).get(&quot;data_array&quot;, [])<br>            else:<br>                raise Exception(&quot;SQL execution failed&quot;)<br>        else:<br>            raise Exception(&quot;Failed to execute SQL command&quot;)<br><br>    # Execute the query and return the result<br>    try:<br>        data = run_sql_query(url, token, query, warehouse_id)<br>    except Exception as e:<br>        return {<br>            &#39;statusCode&#39;: 500,<br>            &#39;body&#39;: json.dumps({&#39;message&#39;: &quot;Error executing the query&quot;, &#39;error&#39;: str(e)})<br>        }<br><br>    return {<br>        &#39;statusCode&#39;: 200,<br>        &#39;body&#39;: json.dumps({&#39;message&#39;: &quot;Completed the query!&quot;, &#39;data&#39;: data})<br>    }</pre><p>Get:</p><pre>import json<br>import os<br>import boto3<br>from databricks import sql<br>from databricks.sdk.core import oauth_service_principal, Config<br> <br># Function to fetch secrets from AWS Secrets Manager<br>def get_secrets(secret_name):<br>    secrets_client = boto3.client(&#39;secretsmanager&#39;)<br>    secret_response = secrets_client.get_secret_value(SecretId=secret_name)<br>    return json.loads(secret_response[&#39;SecretString&#39;])<br> <br># Secret name in AWS Secrets Manager<br>SECRET_NAME = &quot;lambda_integration_databricks_secrets&quot;<br> <br># Fetch secrets<br>secrets = get_secrets(SECRET_NAME)<br>client_id = secrets[&#39;databricks_client_id&#39;]<br>client_secret = secrets[&#39;databricks_client_secret&#39;]<br>server_hostname = secrets[&#39;databricks_server_hostname&#39;]<br>sql_warehouse_http_path=secrets[&#39;databricks_sql_warehouse_http_path&#39;]<br> <br># Logger setup (replace &#39;print&#39; with actual logging if needed)<br>def log(message):<br>    print(f&quot;[LOG] {message}&quot;)<br> <br>def lambda_handler(event, context):<br>    print(&quot;Received event: &quot; + json.dumps(event))<br>    # Credential provider for Databricks connection<br>    def credential_provider():<br>        config = Config(<br>            host=f&quot;https://{server_hostname}&quot;,<br>            client_id=client_id,<br>            client_secret=client_secret,<br>        )<br>        return oauth_service_principal(config)<br>    log(&quot;Fetching email from the event&quot;)<br>    #email=event.get(email,&#39;jl#############.com.test&#39;)<br>    email = event.get(&#39;params&#39;, {}).get(&#39;querystring&#39;, {}).get(&#39;email&#39;)<br>    if not email:<br>        log(&quot;Email input is missing&quot;)<br>        return {<br>            &#39;statusCode&#39;: 200,<br>            &#39;body&#39;: {&#39;message&#39;: &quot;Email input is missing&quot;}<br>        }<br> <br>    log(&quot;Preparing SQL queries&quot;)<br>    query_triton_individual = f&quot;&quot;&quot;<br>    SELECT email_address, name, phone_number, birth_year, gender_cd,zip_code<br>    FROM prod.triton_individual<br>    WHERE email_address = &#39;{email}&#39;<br>    &quot;&quot;&quot;<br>    query_dim_profile = f&quot;&quot;&quot;<br>    SELECT email_address, name, phone_number, birth_year, gender_cd,zip_code<br>    FROM silver.dim_profile<br>    WHERE email_address = &#39;{email}&#39;<br>    &quot;&quot;&quot;<br> <br>    # Function to run SQL query<br>    def run_sql_query(connection, query):<br>        log(f&quot;Executing query: {query}&quot;)<br>        cursor = connection.cursor()<br>        cursor.execute(query)<br>        result = cursor.fetchall()<br>        cursor.close()<br>        return result<br> <br>    try:<br>        log(&quot;Establishing connection to Databricks&quot;)<br>        with sql.connect(<br>            server_hostname=server_hostname,<br>            http_path=sql_warehouse_http_path,<br>            credentials_provider=credential_provider,<br>        ) as connection:<br>            log(&quot;Running triton_individual and dim_profile queries&quot;)<br>            data_triton_individual = run_sql_query(connection, query_triton_individual)<br>            data_dim_profile = run_sql_query(connection, query_dim_profile)<br>            combined_data = data_triton_individual + data_dim_profile<br>    except Exception as e:<br>        log(f&quot;Error executing the query: {str(e)}&quot;)<br>        return {<br>            &#39;statusCode&#39;: 500,<br>            &#39;body&#39;: {&#39;message&#39;: &quot;Error executing the query&quot;, &#39;error&#39;: str(e)}<br>        }<br> <br>    columns = [&quot;email_address&quot;, &quot;name&quot;, &quot;phone_number&quot;, &quot;birth_year&quot;, &quot;gender_cd&quot;, &quot;zip_code&quot;]<br>    result_data = [dict(zip(columns, record)) for record in combined_data]<br> <br>    if not result_data:<br>        log(&quot;No data found for the given email&quot;)<br>        return {<br>            &#39;statusCode&#39;: 200,<br>            &#39;body&#39;: {}<br>        }<br> <br>    log(&quot;Query completed successfully&quot;)<br>    return {<br>        &#39;statusCode&#39;: 200,<br>        &#39;body&#39;: {<br>            &#39;message&#39;: &quot;Completed the query!&quot;,<br>            &#39;data&#39;: result_data<br>        }<br>    }</pre><p>Delete</p><pre>import os<br>import json<br>import boto3<br>from datetime import datetime<br>from databricks import sql<br>from databricks.sdk.core import oauth_service_principal, Config<br> <br># Function to fetch secrets from AWS Secrets Manager<br>def get_secrets(secret_name):<br>    secrets_client = boto3.client(&#39;secretsmanager&#39;)<br>    secret_response = secrets_client.get_secret_value(SecretId=secret_name)<br>    return json.loads(secret_response[&#39;SecretString&#39;])<br> <br># Secret name in AWS Secrets Manager<br>SECRET_NAME = &quot;lambda_integration_databricks_secrets&quot;<br> <br># Fetch secrets<br>secrets = get_secrets(SECRET_NAME)<br>client_id = secrets[&#39;databricks_client_id&#39;]<br>client_secret = secrets[&#39;databricks_client_secret&#39;]<br>server_hostname = secrets[&#39;databricks_server_hostname&#39;]<br>sql_warehouse_http_path = secrets[&#39;databricks_sql_warehouse_http_path&#39;]<br> <br># Logger setup (replace &#39;print&#39; with actual logging if needed)<br>def log(message):<br>    print(f&quot;[LOG] {message}&quot;)<br> <br>def lambda_handler(event, context):<br>    print(&quot;Received event: &quot; + json.dumps(event))<br>   <br>    # Credential provider for Databricks connection<br>    def credential_provider():<br>        config = Config(<br>            host=f&quot;https://{server_hostname}&quot;,<br>            client_id=client_id,<br>            client_secret=client_secret,<br>        )<br>        return oauth_service_principal(config)<br> <br>    log(&quot;Fetching email from the event&quot;)<br>    email = event.get(&#39;params&#39;, {}).get(&#39;querystring&#39;, {}).get(&#39;email&#39;)<br>    if not email:<br>        log(&quot;Email input is missing&quot;)<br>        return {<br>            &#39;statusCode&#39;: 200,<br>            &#39;body&#39;: {&#39;message&#39;: &quot;Email input is missing&quot;}<br>        }<br>    # Replacement string with current date and time<br>    current_datetime = datetime.now().strftime(&#39;%Y-%m-%d %H:%M:%S.%f&#39;)[:-3]<br>    replacement_string = f&quot;[REDACTED]-Request Date {current_datetime}&quot;<br>   <br>    # Current date and time for etl-date<br>    etl_date = datetime.now().strftime(&#39;%Y-%m-%dT%H:%M:%S.%f&#39;)[:-3]<br> <br>    # SQL Queries to replace data and update etl-date<br>    queries_to_update = [<br>        f&quot;&quot;&quot;<br>        UPDATE silver.dim_profile<br>        SET email_address = &#39;{replacement_string}&#39;, name = &#39;{replacement_string}&#39;, phone_number = &#39;{replacement_string}&#39;, etl_date = &#39;{etl_date}&#39;<br>        WHERE email_address = &#39;{email}&#39;<br>        &quot;&quot;&quot;,<br>        f&quot;&quot;&quot;<br>        UPDATE silver.id_resolution_send_history<br>        SET email_address = &#39;{replacement_string}&#39;, phone_number = &#39;{replacement_string}&#39;, etl_date = &#39;{etl_date}&#39;<br>        WHERE email_address = &#39;{email}&#39;<br>        &quot;&quot;&quot;,<br>        f&quot;&quot;&quot;<br>        UPDATE prod.triton_individual<br>        SET email_address = &#39;{replacement_string}&#39;, name = &#39;{replacement_string}&#39;, phone_number = &#39;{replacement_string}&#39;, etl_date = &#39;{etl_date}&#39;<br>        WHERE email_address = &#39;{email}&#39;<br>        &quot;&quot;&quot;<br>    ]<br> <br>    try:<br>        log(&quot;Establishing connection to Databricks&quot;)<br>        with sql.connect(<br>            server_hostname=server_hostname,<br>            http_path=sql_warehouse_http_path,<br>            credentials_provider=credential_provider,<br>        ) as connection:<br>            cursor = connection.cursor()<br> <br>            # Execute update queries<br>            for query in queries_to_update:<br>                log(f&quot;Executing query: {query}&quot;)<br>                cursor.execute(query)<br> <br>            # Additional logic for prod.individual_id_res_link<br>            query_individual_id_res_link = f&quot;&quot;&quot;<br>            SELECT triton_individual_guid<br>            FROM prod.individual_id_res_link<br>            WHERE id_resolution_link_id = &#39;{email}&#39; AND id_resolution_link_type_cd = &#39;EM&#39;<br>            &quot;&quot;&quot;<br> <br>            cursor.execute(query_individual_id_res_link)<br>            results = cursor.fetchall()<br> <br>            if not results:<br>                log(&quot;No data found for the given email&quot;)<br>            else:<br>                triton_individual_guid = results[0][0]<br> <br>                # Update email and phone number in prod.individual_id_res_link and set etl-date<br>                update_queries_individual_id_res_link = [<br>                    f&quot;&quot;&quot;<br>                    UPDATE prod.individual_id_res_link<br>                    SET id_resolution_link_id = &#39;{replacement_string}&#39;, etl_date = &#39;{etl_date}&#39;<br>                    WHERE triton_individual_guid = &#39;{triton_individual_guid}&#39; AND id_resolution_link_type_cd = &#39;EM&#39;<br>                    &quot;&quot;&quot;,<br>                    f&quot;&quot;&quot;<br>                    UPDATE prod.individual_id_res_link<br>                    SET id_resolution_link_id = &#39;{replacement_string}&#39;, etl_date = &#39;{etl_date}&#39;<br>                    WHERE triton_individual_guid = &#39;{triton_individual_guid}&#39; AND id_resolution_link_type_cd = &#39;PH&#39;<br>                    &quot;&quot;&quot;<br>                ]<br> <br>                for query in update_queries_individual_id_res_link:<br>                    log(f&quot;Executing query: {query}&quot;)<br>                    cursor.execute(query)<br> <br>            cursor.close()<br>            connection.close()<br> <br>        log(&quot;Update completed successfully&quot;)<br>        return {<br>            &#39;statusCode&#39;: 200,<br>            &#39;body&#39;: {&#39;message&#39;: &quot;success&quot;}<br>        }<br> <br>    except Exception as e:<br>        log(f&quot;Error executing the query: {str(e)}&quot;)<br>        return {<br>            &#39;statusCode&#39;: 500,<br>            &#39;body&#39;: {&#39;message&#39;: &quot;Error executing the query&quot;, &#39;error&#39;: str(e)}<br>        }</pre><p>terraform</p><pre>1 delete/forget email &quot;Listening logs can be maintained.<br><br>Scramble (make non-unique): email, name, phone, TU ID, IPs &amp; MAIDs - that directly matched on Profile IDs associated with the email. Indirect/probablistic matched IPs, MAIDs need not be scrambled. No need to scramble gender and birth year.<br><br>Check with Spencer, and if there is a need to keep IP or MAID, a risk assessment will need to be done with Andrew&#39;s help.&quot; Do case-insensitive match on email and find all profile_ids for that email then scrub. Also set is_email_valid, is_IP_valid, is_phone_valied, is_MAID_valid all to false. &quot;Create our own, no common spec<br>Need secure API<br>JSON response preferred&quot; 15-20 per month Yes<br><br><br><br><br><br>write a secure API JSON response preferred that caters to lambda function that delete/forget, email here are more details <br>Scramble (make non-unique): email, name, phone, TU ID, IPs &amp; MAIDs - <br>that directly matched on Profile IDs associated with the email. <br>Indirect/probablistic matched IPs, MAIDs need not be scrambled.<br> No need to scramble gender and birth year.<br> Scramble (make non-unique): email, name, phone, TU ID, IPs &amp; MAIDs - <br> that directly matched on Profile IDs associated with the email.<br> Indirect/probablistic matched IPs, MAIDs need not be scrambled. No need to scramble gender and birth year.<br> <br> also we need to setup the I want to update Infrastructure as Code scripts to provision API gateway and LAMDA services for CCPA which we request via API<br><br>I want to update the roles, permissions, and policies so that:<br><br>OneTrust can access the API Gateway to submit CCPA requests (delete and get).<br><br>API Gateway can forward the requests to appropriate Lambda function.<br><br>Lambda functions can connect and update/view data in Databricks.<br><br>So that we can provision the application components in higher environments.<br>--------<br><br><br>this are the files that we need to update for above.<br> first file = iam_role.tf<br><br># Audience Platform IAM Roles:<br><br># IAM Role for IHR DE S3 Readonly (IHM datawarehouse bucket with source listener data)<br>resource &quot;aws_iam_role&quot; &quot;audience_platform_ihr_de_s3_readonly&quot; {<br>  name = &quot;audience-platform-ihr-de-s3-readonly&quot;<br>  assume_role_policy = jsonencode({<br>    Version : &quot;2012-10-17&quot;,<br>    Statement : [<br>      {<br>        Effect : &quot;Allow&quot;,<br>        Principal : {<br>          Service : &quot;ec2.amazonaws.com&quot;<br>        },<br>        Action : &quot;sts:AssumeRole&quot;<br>      }<br>    ]<br>  })<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}<br><br># Attach AmazonS3FullAccess Policy to the IHR DE S3 Readonly Role<br>resource &quot;aws_iam_role_policy_attachment&quot; &quot;ihr_de_s3_readonly_s3_full_access&quot; {<br>  role       = aws_iam_role.audience_platform_ihr_de_s3_readonly.name<br>  policy_arn = &quot;arn:aws:iam::aws:policy/AmazonS3FullAccess&quot;<br>}<br><br># IAM Role for the Lambda Encryption Service<br>resource &quot;aws_iam_role&quot; &quot;audience_platform_lambda_encryption_service_role&quot; {<br>  name = &quot;audience-platform-lambda-encryption-service-role&quot;<br>  assume_role_policy = jsonencode({<br>    Version : &quot;2012-10-17&quot;,<br>    Statement : [<br>      {<br>        Effect : &quot;Allow&quot;,<br>        Principal : {<br>          Service : &quot;lambda.amazonaws.com&quot;<br>        },<br>        Action : &quot;sts:AssumeRole&quot;<br>      }<br>    ]<br>  })<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}<br><br># Custom S3 Policy for Limited Bucket Access<br>resource &quot;aws_iam_policy&quot; &quot;lambda_s3_limited_access_policy&quot; {<br>  name        = &quot;lambda-s3-limited-access-policy&quot;<br>  description = &quot;Policy to allow limited S3 access for Lambda function&quot;<br>  policy = jsonencode({<br>    Version : &quot;2012-10-17&quot;,<br>    Statement : [<br>      {<br>        Effect : &quot;Allow&quot;,<br>        Action : [<br>          &quot;s3:GetObject&quot;,<br>          &quot;s3:PutObject&quot;,<br>          &quot;s3:DeleteObject&quot;,<br>          &quot;s3:GetBucketLocation&quot;,<br>          &quot;s3:ListBucket&quot;<br>        ],<br>        Resource : [<br>          &quot;arn:aws:s3:::triton-audience-platform-delta-lake-${var.environment}&quot;,<br>          &quot;arn:aws:s3:::triton-audience-platform-delta-lake-${var.environment}/*&quot;,<br>          &quot;arn:aws:s3:::triton-audience-platform-transunion-${var.environment}&quot;,<br>          &quot;arn:aws:s3:::triton-audience-platform-transunion-${var.environment}/*&quot;<br>        ]<br>      }<br>    ]<br>  })<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}<br><br># Attach Custom S3 Policy to the Lambda Encryption Service Role<br>resource &quot;aws_iam_role_policy_attachment&quot; &quot;lambda_encryption_service_s3_limited_access&quot; {<br>  role       = aws_iam_role.audience_platform_lambda_encryption_service_role.name<br>  policy_arn = aws_iam_policy.lambda_s3_limited_access_policy.arn<br>}<br><br># Attach Aws managed secrets manager Policy to the Lambda Encryption Service Role<br>resource &quot;aws_iam_role_policy_attachment&quot; &quot;lambda_encryption_service_secrets_manager_access&quot; {<br>  role       = aws_iam_role.audience_platform_lambda_encryption_service_role.name<br>  policy_arn = &quot;arn:aws:iam::aws:policy/SecretsManagerReadWrite&quot;<br>}<br><br># Attach Aws managed cloud watch Policy to the Lambda Encryption Service Role<br>resource &quot;aws_iam_role_policy_attachment&quot; &quot;lambda_encryption_service_cloudwatch_access&quot; {<br>  role       = aws_iam_role.audience_platform_lambda_encryption_service_role.name<br>  policy_arn = &quot;arn:aws:iam::aws:policy/CloudWatchFullAccess&quot;<br>}<br><br># Add a policy to the existing IAM role with the necessary permissions<br>resource &quot;aws_iam_policy&quot; &quot;lambda_tag_policy&quot; {<br>  name        = &quot;lambda_tag_policy&quot;<br>  description = &quot;Policy to allow tagging of Lambda functions&quot;<br>  policy = jsonencode({<br>    Version = &quot;2012-10-17&quot;,<br>    Statement = [<br>      {<br>        Action = [<br>          &quot;lambda:TagResource&quot;,<br>          &quot;lambda:CreateFunction&quot;,              # Adding CreateFunction if not already present<br>          &quot;lambda:UpdateFunctionConfiguration&quot;, # Adding UpdateFunctionConfiguration if not already present<br>        ],<br>        Effect   = &quot;Allow&quot;,<br>        Resource = &quot;*&quot;<br>      }<br>    ]<br>  })<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}<br><br># Attach the policy to the IAM role<br>resource &quot;aws_iam_role_policy_attachment&quot; &quot;audience_platform_lambda_tag_policy_attachment&quot; {<br>  role       = aws_iam_role.audience_platform_lambda_encryption_service_role.name<br>  policy_arn = aws_iam_policy.lambda_tag_policy.arn<br>}<br><br># iam_roles output for the lambda input<br>output &quot;audience_platform_lambda_encryption_service_role_arn&quot; {<br>  value = aws_iam_role.audience_platform_lambda_encryption_service_role.arn<br>}<br><br><br>2nd file= terraform.tf<br><br>terraform {<br>  required_version = &quot;&gt;= 1.0.0&quot;<br><br>  required_providers {<br>    aws = {<br>      source  = &quot;hashicorp/aws&quot;<br>      version = &quot;~&gt; 5.0&quot;<br>    }<br>  }<br>}<br><br>3rd file = variables.tf<br><br>variable &quot;environment&quot; {<br>  description = &quot;Environment name (dev, staging, prod)&quot;<br>  type        = string<br>}<br><br>variable &quot;product&quot; {<br>  description = &quot;Product name&quot;<br>  type        = string<br>  default     = &quot;audience-platform&quot;<br>}<br><br>variable &quot;repo&quot; {<br>  description = &quot;Repo name&quot;<br>  type        = string<br>  default     = &quot;https://github.com/triton-digital-em/tf-audience-platform-etl-automation&quot;<br>}<br><br>variable &quot;team&quot; {<br>  description = &quot;Team name&quot;<br>  type        = string<br>  default     = &quot;sapient&quot;<br>}<br><br><br>4th file = lambda.tf<br><br># Define the Encryption Lambda function<br>resource &quot;aws_lambda_function&quot; &quot;lambda_encryption_service&quot; {<br>  filename         = &quot;${path.module}/encryption_func.zip&quot; # Path to the encryption .zip file<br>  function_name    = &quot;tu_encryption_service&quot;<br>  role             = var.iam_role_arn<br>  handler          = &quot;lambda_function.lambda_handler&quot; # Adjust handler as per your Lambda function<br>  runtime          = &quot;python3.9&quot;                      # Adjust runtime as per your Lambda function<br>  source_code_hash = filebase64sha256(&quot;${path.module}/encryption_func.zip&quot;)<br>  timeout          = 900   # 15 minutes timeout in seconds<br>  memory_size      = 10240 # Memory size in MB (10GB)<br><br>  # Configure ephemeral storage<br>  ephemeral_storage { size = 10240 } # 10 GB<br><br>  environment {<br>    variables = {<br>      S3_BUCKET            = &quot;triton-audience-platform-delta-lake-${var.environment}&quot;<br>      ARCHIVE_PREFIX       = &quot;encrypted_archive/&quot;<br>      TO_TRANSUNION_BUCKET = &quot;triton-audience-platform-transunion-${var.environment}&quot;<br>      TO_TRANSUNION_PREFIX = &quot;to_transunion/&quot;<br>    }<br>  }<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}<br><br># Define the Decryption Lambda function<br>resource &quot;aws_lambda_function&quot; &quot;lambda_decryption_service&quot; {<br>  filename         = &quot;${path.module}/decryption_func.zip&quot; # Path to the decryption .zip file<br>  function_name    = &quot;tu_decryption_service&quot;<br>  role             = var.iam_role_arn<br>  handler          = &quot;lambda_function.lambda_handler&quot; # Adjust handler as per your Lambda function<br>  runtime          = &quot;python3.9&quot;                      # Adjust runtime as per your Lambda function<br>  source_code_hash = filebase64sha256(&quot;${path.module}/decryption_func.zip&quot;)<br>  timeout          = 900   # 15 minutes timeout in seconds<br>  memory_size      = 10240 # Memory size in MB (10GB)<br><br>  # Configure ephemeral storage<br>  ephemeral_storage { size = 10240 } # 10 GB<br><br>  environment {<br>    variables = {<br>      ARCHIVE_PREFIX                   = &quot;archive_decrypt/&quot;<br>      DECRYPTED_BUCKET                 = &quot;triton-audience-platform-delta-lake-${var.environment}&quot;<br>      ENCRYPTED_BUCKET                 = &quot;triton-audience-platform-transunion-${var.environment}&quot;<br>      FROM_TRANSUNION_DECRYPTED_PREFIX = &quot;from_transunion_decrypted/&quot;<br>      FROM_TRANSUNION_PREFIX           = &quot;from_transunion/&quot;<br>    }<br>  }<br><br>  tags = {<br>    env     = var.environment<br>    iac     = &quot;true&quot;<br>    product = var.product<br>    repo    = var.repo<br>    team    = var.team<br>  }<br>}</pre><p>Reservation_ingestion:</p><pre>datasource<br>entity<br>exceptions<br>helper<br>documentdbSinkHelper<br>ReservationCollationHelper<br>ReservationEventHelper<br>ReservationEventsCBHelper<br>ReservationFiltrationHelper<br>ReservationIngestionHelper<br>jobs<br><br><br>datasource<br>entity<br><br>DocumentDBSINK:<br><br><br>package com.marriott.mgpapp.spark.reservation.ingestion.datasource<br><br>import com.marriott.elevate.spark.framework.config.MessageBrokerDetails<br>import com.marriott.elevate.spark.framework.utils.JsonUtil.toJson<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion.exceptions.{CustomException, DocumentDBException}<br>import com.marriott.mgpapp.spark.reservation.ingestion.helper.DocumentDBSinkHelper<br>import com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationEventsCBHelper.getMongoClient<br>import com.marriott.utility.CustomLogger<br>import com.mongodb.client.MongoClient<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.SparkConf<br>import org.apache.spark.sql.ForeachWriter<br>import org.bson.codecs.configuration.CodecRegistries.{fromProviders, fromRegistries}<br>import org.mongodb.scala.MongoClient.DEFAULT_CODEC_REGISTRY<br>import org.mongodb.scala.bson.Document<br>import org.mongodb.scala.bson.codecs.Macros._<br>import org.mongodb.scala.model.ReplaceOptions<br><br>import java.time.format.DateTimeParseException<br>import java.time.{LocalDate, ZoneId}<br>import java.util.{Calendar, Date}<br>import javax.sql.DataSource<br>import scala.util.{Failure, Success, Try}<br><br>class DocumentDBKafkaSink(conf: SparkConf,<br>                          kafkaConfig: MessageBrokerDetails, kafkamaxsize: Float)<br>  extends ForeachWriter[ReservationEventsEntity] with DocumentDBSinkHelper with LazyLogging with Serializable {<br><br>  //  implicit val jobConfig: JobConfigurationDetails = jobConfigDetails<br>  val jobIdentifier: String = ReservationIngestion<br>  val jobName: String = Reservations<br>  val log: CustomLogger = new CustomLogger(&quot;&quot;)<br>  val appNameForLogger = appName<br><br>  @transient var dataSource: DataSource = _<br>  @transient var mongodbClient: MongoClient = _<br>  //  @transient var kafkaProducer:KafkaProducer[String,String]= _<br><br>  override def open(partitionId: Long, version: Long): Boolean = {<br><br>    log.info(&quot;Opening DocumentDB connections - DocumentDBKafkaSink.open()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    mongodbClient = getMongoClient(conf, DocumentDBInput)<br>    true<br>  }<br><br>  def getEpochOfDepartureDatePlus60Days(departureDateString: String): Date = {<br>    try {<br>      val departureDate = LocalDate.parse(departureDateString)<br>      val expirationDate = departureDate.plusDays(60)<br>      Date.from(expirationDate.atStartOfDay(ZoneId.systemDefault()).toInstant())<br>    } catch {<br>      case e: DateTimeParseException =&gt;<br>        logger.error(s&quot;Error parsing &#39;departureDateString&#39;: ${e.getMessage}&quot;)<br>        new CustomException(e, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        val calendar = Calendar.getInstance()<br>        val currentDate = calendar.getTime()<br>        // Add 60 days to the current date<br>        calendar.add(Calendar.DAY_OF_MONTH, 0)<br>        calendar.getTime()<br>    }<br>  }<br><br>  override def process(record: ReservationEventsEntity): Unit = {<br><br>    log.info(&quot;Processing record into documentDB - DocumentDBKafkaSink.process()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //println(&quot;5 Aug - DBsink Process | input record - &quot; + record)<br><br>    val codecRegistry = fromRegistries(fromProviders(classOf[HotelReservationWOS3]), DEFAULT_CODEC_REGISTRY)<br>    val collection = mongodbClient.getDatabase(conf.get(DocumentDbInputDatabase)).getCollection(conf.get(DocumentDbInputCollection)).withCodecRegistry(codecRegistry)<br><br>    if (record.newReservation.isDefined) {<br>      var newReswithUpdatedInv = if (record.oldReservation.isDefined) {<br>        var ProductUseMap = record.oldReservation.get.segments.head.offer.get.productUses.map(pu =&gt; {<br>          val orgInvTypeCode = pu.inventoryTypeCode<br>          pu.timespan.get.start -&gt; orgInvTypeCode<br>        }).toMap<br>        val resWithOrgInv = record.newReservation.get.copy(segments = record.newReservation.get.segments.map(seg =&gt; seg.copy(offer = Option(record.newReservation.get.segments.head.offer.get.copy(productUses = record.newReservation.get.segments.head.offer.get.productUses.map(productUse =&gt; productUse.copy(originalInventoryTypeCode = ProductUseMap.getOrElse(productUse.timespan.get.start,None))))))))<br>        resWithOrgInv<br>      }<br>      else {<br>        val resWithOrgInv = record.newReservation.get.copy(segments = record.newReservation.get.segments.map(seg =&gt; seg.copy(offer = Some(record.newReservation.get.segments.head.offer.get.copy(productUses = record.newReservation.get.segments.head.offer.get.productUses.map(productUse =&gt; productUse.copy(originalInventoryTypeCode = productUse.inventoryTypeCode)))))))<br>        resWithOrgInv<br>      }<br><br>      val departureDateString = record.newReservation.get.timespan.get.end<br>      //      val document = Document(toJson(<br>      //        HotelReservationWOS3DocDb(record.reservationKey.get, record.newReservation.get)<br>      //      )).append(&quot;expiresAt&quot;, getEpochOfDepartureDatePlus60Days(departureDateString))<br>      val document = Document(toJson(<br>        HotelReservationWOS3DocDb(record.reservationKey.get, newReswithUpdatedInv)<br>      )).append(&quot;expiresAt&quot;, getEpochOfDepartureDatePlus60Days(departureDateString))<br>      Try {<br>        collection.replaceOne(<br>          Document(&quot;_id&quot; -&gt; record.reservationKey),<br>          document,<br>          ReplaceOptions().upsert(true)<br>        )<br>        val updatedReservationKey = record.reservationKey.get.split(&quot;_&quot;).toList match {<br>          case cfNum :: date :: Nil if cfNum.length &gt;= 8 =&gt; s&quot;${cfNum.take(8)}_$date&quot;<br>        }<br>        if(record.reservationKey.get != updatedReservationKey) {<br>          collection.deleteOne(Document(&quot;_id&quot; -&gt; updatedReservationKey))<br>        }<br>      }<br>      match {<br>        case Success(_) =&gt; closeSession<br>          log.info(&quot;Record has been processed successfully - DocumentDBKafkaSink.process()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        case Failure(e) =&gt; closeSession<br>          log.error(&quot;Error while processing the record to couchbase :: Exception :&quot; + e.printStackTrace(),<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>          new DocumentDBException(e, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      }<br>    }<br>  }<br><br>  def closeSession(): Unit = {<br>    log.info(&quot;Closing DB Session : closeSession()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    mongodbClient.close()<br>  }<br><br>  override def close(errorOrNull: Throwable): Unit = {<br>    closeSession<br>  }<br>}<br><br>entity:<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.entity<br><br>import com.fasterxml.jackson.module.scala.JsonScalaEnumeration<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{EmptyString, ErrorTrue, IrrecoverableError}<br>import com.marriott.elevate.spark.framework.entity.{BaseEvent, DefaultEventInfo, PayLoad, StateInput}<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.ActionStatusEnum.ActionStatusEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.AgeQualifyingCodeEnum.AgeQualifyingCodeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.BaseChargeFrequencyEnum.BaseChargeFrequencyEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.ChargeFrequencyEnum.ChargeFrequencyEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.DaysOfWeekEnum.DaysOfWeekEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.DeadlineTypeEnum.DeadlineTypeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.DepositPolicyTypeEnum.DepositPolicyTypeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.FeeTypeEnum.FeeTypeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.GuaranteePolicyTypeEnum.GuaranteePolicyTypeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.OtaCodeEnum.OtaCodeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.PaymentCodeEnum.PaymentCodeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.PaymentSubTypeCodeEnum.PaymentSubTypeCodeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.PricingFrequencyEnum.PricingFrequencyEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.RatePlanTypeEnum.RatePlanTypeEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.RequestStatusEnum.RequestStatusEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationStatusEnum.ReservationStatusEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.SegmentStatusEnum.SegmentStatusEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.TimePeriodEnum.TimePeriodEnum<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.TypeEnum.TypeEnum<br>import com.marriott.model.CustomMetadata<br><br>import java.sql.{Date, Timestamp}<br>import java.time.LocalDateTime<br>import java.time.format.DateTimeFormatter<br>import scala.reflect.runtime.{universe =&gt; ru}<br><br>final case class CustomloggerMetadata(traceId: String,<br>                                      spanId: String,<br>                                      project: String,<br>                                      app: String,<br>                                      flow: String,<br>                                      customData: Map[String, String]<br>                                     ) extends CustomMetadata with Serializable<br><br>case class HotelReservation(committedSequenceNumber: Option[Integer],<br>                            creator: Option[BookingChannel],<br>                            dateTimeStamp: Option[DateTimeStamp],<br>                            formOfPayment: Option[FormOfPayment],<br>                            hotels: Seq[Hotel],<br>                            products: Seq[InventoryTypeDetails],<br>                            ratePlans: Seq[RatePlanDetails],<br>                            requestor: Option[BookingChannel],<br>                            @JsonScalaEnumeration(classOf[ReservationStatusEnumType]) reservationStatusEnum: ReservationStatusEnum,<br>                            rewardBenefitInvokedInd: Option[Boolean],<br>                            segments: Seq[Segment],<br>                            timespan: Option[Timespan],<br>                            totalReservation: Seq[GrossAmount],<br>                            travelers: Seq[Traveler],<br>                            confirmationNumber: ProviderStateID,<br>                            totalGrossRevenue: Option[Double],<br>                            totalNetRevenue: Option[Double],<br>                            groupContracts: Option[Seq[Option[GroupContract]]],<br>                            sourceCRSEnum: String)<br><br>case class GroupContract(code: Option[String],<br>                         groupIntermediaryID: Option[String],<br>                         confirmationNumber: ProviderStateID,<br>                         externalIds: Option[Seq[GroupExternalItemId]],<br>                         groupType: Option[String])<br><br>case class GroupExternalItemId(id: Option[String],<br>                               `type`: Option[String])<br><br>case class PropertyBrand(brandName: Option[String],<br>                         code: Option[String])<br><br>case class PropertyCode(hotelName: Option[String],<br>                        code: String)<br><br>case class ProviderStateID(provider: Option[String],<br>                           value: String,<br>                           `type`: Option[String])<br><br>case class BookingChannel(companyName: Option[String],<br>                          origin: Option[String],<br>                          agencyInfo: Option[Seq[AgencyInformation]],<br>                          locationCode: Option[String])<br><br>case class Segment(@JsonScalaEnumeration(classOf[ActionStatusEnumType]) actionStatusEnum: Option[ActionStatusEnum],<br>                   assignedRooms: Option[Seq[AllocatedRoom]],<br>                   offer: Option[Offer],<br>                   owners: Option[Seq[OwnerDetails]],<br>                   @JsonScalaEnumeration(classOf[SegmentStatusEnumType]) segmentStatusEnum: Option[SegmentStatusEnum],<br>                   payDetails: Option[Seq[PayStayDetails]],<br>                   stayDetails: Option[PayStayDetails],<br>                   specialRequests: Option[Seq[SpecialRequest]],<br>                   taxAndFeeDefinitions: Option[Seq[TaxDetails]],<br>                   timespan: Option[Timespan],<br>                   totalSegment: Option[CurrencyAmount],<br>                   capsAndRestriction: Option[CapsAndRestriction],<br>                   sleepingRoomInd: Boolean<br>                  )<br><br><br>case class DateTimeStamp(create: Option[String],<br>                         creationDateTimeHotelLocalTime: Option[String],<br>                         purge: Option[String],<br>                         lastModify: Option[String],<br>                         lastUpdateDateTimeHotelLocalTime: Option[String])<br><br>case class Hotel(propertyBrand: PropertyBrand,<br>                 propertyCode: PropertyCode,<br>                 propertyCodeContext: Option[String],<br>                 propertyCurrency : Option[String]<br>                )<br><br>case class InventoryTypeDetails(dspRoomTypeCode: Option[String],<br>                                inventoryTypeCode: String,<br>                                miosRecommendedRoomTypeCode: Option[String],<br>                                pmsRoomTypeCode: Option[String],<br>                                roomCategory: Option[String])<br><br><br>case class RatePlanDetails(code: String,<br>                           advanceBookingRestrictions: Option[Seq[AdvanceBookingRestriction]],<br>                           commissionStatusType: Option[String],<br>                           details: Details,<br>                           marketType: Option[MarketType],<br>                           packageProducts: Option[Seq[PackageProduct]],<br>                           descriptions: Option[Seq[Description]],<br>                           ratePlanCategory: Option[String],<br>                           ratePlanID: Option[String],<br>                           ratePlanSource: Option[String],<br>                           @JsonScalaEnumeration(classOf[RatePlanTypeEnumType]) ratePlanTypeEnum: Option[Seq[RatePlanTypeEnum]])<br><br>case class MarketType(marketCode: Option[String],<br>                      description: Option[Description],<br>                      commissionableInd: Option[Boolean])<br>case class PackageProduct(<br>                           priceGridCode: Option[String]<br>                         )<br><br>case class AdvanceBookingRestriction(offset: Option[Range])<br><br>case class Timespan(start: String,<br>                    end: String,<br>                    duration: Option[String])<br><br>case class GrossAmount(rateAmount: RateAmount,<br>                       totalTaxes: CurrencyAmount)<br><br>case class Traveler(<br>                     loyaltyPrograms: Option[Seq[LoyaltyProgram]],<br>                     personIdentifiers: Option[Seq[PersonIdentifier]])<br><br>case class FormOfPayment(cash: Option[Seq[CashPayment]],<br>                         directBill: Option[Seq[DirectBill]],<br>                         loyaltyCertificate: Option[Seq[LoyaltyCertificate]],<br>                         otherPaymentForms: Option[Seq[OtherPaymentForm]],<br>                         paymentCard: Option[Seq[PaymentCard]],<br>                         voucher: Option[Seq[Voucher]])<br><br>case class AgencyInformation(agentInfo: Option[AgentInformation])<br><br>case class OwnerDetails(agencyInformation: Option[AgencyInformation])<br><br>case class AllocatedRoom(allocatedRoom: Option[String])<br><br>case class Offer(productUses: Seq[ProductUse],<br>                 termsAndConditions: Option[TermsAndConditions])<br><br><br>case class TaxDetails(feeInd: Option[Boolean],<br>                      periods: Option[Seq[TaxPeriod]],<br>                      @JsonScalaEnumeration(classOf[ChargeFrequencyEnumType]) chargeFrequencyEnum: Option[ChargeFrequencyEnum],<br>                      description: Option[Seq[Description]],<br>                      @JsonScalaEnumeration(classOf[TypeEnumType]) typeEnum: Option[TypeEnum])<br><br>case class TaxPeriod(value: Option[Seq[TaxValue]])<br><br>case class TaxValue(amountValue: Option[CurrencyAmount],<br>                    age: Option[Range])<br><br>case class Range(min: Option[String],<br>                 max: Option[String])<br><br>case class Details(commissionableInd: Option[Boolean] = Some(false),<br>                   propertyMarketSegmentation: Option[String])<br><br><br>case class RateAmount(amtAfTx: Option[CurrencyAmount],<br>                      amtBfTx: Option[CurrencyAmount],<br>                      bsAmt: Option[CurrencyAmount])<br><br>case class LoyaltyProgram(loyaltyLevel: Option[String],<br>                          programCode: Option[ProgramCode])<br><br>case class CashPayment(paymentTypeCodeInfo: PaymentTypeCodeRef,<br>                       remark: Option[String])<br><br>case class LoyaltyCertificate(certificateNumber: String,<br>                              loyaltyProgram: LoyaltyProgram,<br>                              paymentTypeCodeInfo: PaymentTypeCodeRef)<br><br>case class PaymentCard(paymentTypeCodeInfo: PaymentTypeCodeRef,<br>                       description: Option[Description])<br><br>case class SpecialRequest(base: Option[Base],<br>                          codeContext: Option[String] = Some(&quot;Marriott&quot;),<br>                          description: Option[Seq[Description]],<br>                          guestViewableInd: Option[Boolean] = Some(false),<br>                          includeInGRTInd: Option[Boolean] = Some(false),<br>                          inherentInRoomTypeInd: Option[Boolean] = Some(false),<br>                          maxAllowed: Option[Integer] = Some(100),<br>                          @JsonScalaEnumeration(classOf[OtaCodeEnumType]) otaCodeEnum: Option[OtaCodeEnum],<br>                          requestCode: Option[String],<br>                          @JsonScalaEnumeration(classOf[RequestStatusEnumType]) requestStatusEnum: Option[RequestStatusEnum],<br>                          roomNumber: Option[Integer])<br><br>case class PersonIdentifier(value: String)<br><br>case class DirectBill(paymentTypeCodeInfo: PaymentTypeCodeRef,<br>                      description: Description)<br><br>case class OtherPaymentForm(paymentTypeCodeInfo: PaymentTypeCodeRef,<br>                            description: Description)<br><br>case class Voucher(paymentTypeCodeInfo: Option[PaymentTypeCodeRef],<br>                   description: Description)<br><br>case class AgentInformation(agentDutyCode: Option[String],<br>                            agentInitials: Option[String],<br>                            agentSine: Option[String])<br><br>case class ProductUse(guestCounts: Option[Seq[GuestCount]],<br>                      platinumOverrideInd: Option[Boolean],<br>                      productRates: ProductRate,<br>                      groupBlockCode: Option[String],<br>                      ratePlanCode: Option[String],<br>                      priceGridCode: Option[String],<br>                      totalGuestCount: Option[GuestCount],<br>                      inventoryTypeCode: Option[String],<br>                      originalInventoryTypeCode: Option[String],<br>                      quantity: Integer,<br>                      timespan: Option[TimespanWOS3])<br><br>case class TermsAndConditions(cancellationPolicy: Option[Seq[CancellationPolicy]],<br>                              depositPolicy: Option[Seq[DepositPolicy]],<br>                              guaranteePolicy: Option[Seq[GuaranteePolicy]])<br><br>case class Description(text: Option[Seq[String]],<br>                       locale: Option[Locale])<br><br>case class Locale(languageCode: Option[String],<br>                  countryCode: Option[String])<br><br>case class ProgramCode(programId: Option[String])<br><br>case class PaymentTypeCodeRef(@JsonScalaEnumeration(classOf[PaymentSubTypeCodeEnumType]) paymentSubTypeCodeEnum: Option[PaymentSubTypeCodeEnum])<br><br>case class Base(amount: CurrencyAmount,<br>                @JsonScalaEnumeration(classOf[BaseChargeFrequencyEnumType]) chargeFrequencyEnum: Option[BaseChargeFrequencyEnum])<br><br>case class GuestCount(age: Option[Integer],<br>                      @JsonScalaEnumeration(classOf[AgeQualifyingCodeEnumType]) ageQualifyingCodeEnum: Option[AgeQualifyingCodeEnum],<br>                      count: Integer,<br>                      ageBucket: Option[String])<br><br>case class CurrencyAmount(value: BigDecimal,<br>                          currencyCode: Option[String],<br>                          numberOfDecimals: Option[Integer])<br><br>case class AmountCurrencyAmount(value: BigDecimal,<br>                                currencyCode: Option[String],<br>                                numberOfDecimals: Option[Integer])<br><br>case class ProductRate(dailyRates: Seq[RateChange],<br>                       @JsonScalaEnumeration(classOf[PricingFrequencyEnumType]) pricingFrequencyEnum: Option[PricingFrequencyEnum])<br><br>case class DOW_Restrictions(@JsonScalaEnumeration(classOf[DaysOfWeekEnumType]) daysOfWeekEnum: Option[Seq[DaysOfWeekEnum]])<br><br>case class OtherRule(policyCode: PolicyCode)<br><br>case class CancellationPolicy(amountPercent: Option[AmountPercent],<br>                              cancelPolicyInd: Option[Boolean] = Some(false),<br>                              deadline: Option[Deadline],<br>                              nonRefundableInd: Option[Boolean] = Some(false),<br>                              policyNumber: Option[Integer])<br><br>case class DepositPolicy(amountPercent: Option[AmountPercent],<br>                         @JsonScalaEnumeration(classOf[GuaranteePolicyTypeEnumType]) guaranteePolicyTypeEnum: Option[GuaranteePolicyTypeEnum],<br>                         @JsonScalaEnumeration(classOf[DepositPolicyTypeEnumType]) typeEnum: Option[DepositPolicyTypeEnum])<br><br>case class GuaranteePolicy(addressLines: Option[Seq[String]],<br>                           amountPercent: Option[AmountPercent],<br>                           deadline: Option[Deadline],<br>                           guaranteeCode: Option[String],<br>                           guaranteesAccepted: Option[GuaranteesAccepted],<br>                           holdTime: Option[String],<br>                           @JsonScalaEnumeration(classOf[PaymentCodeEnumType]) paymentCodeEnum: Option[PaymentCodeEnum],<br>                           retributionType: Option[String])<br><br>case class RateChange(dailyRate: DailyRate,<br>                      timespan: Option[TimespanWOS3])<br><br>case class PolicyCode(codeContext: Option[String],<br>                      policyId: Option[String],<br>                      qualifiedRateYNInd: Option[Boolean])<br><br>case class Deadline(absoluteDateTime: Option[String],<br>                    @JsonScalaEnumeration(classOf[DeadlineTypeEnumType]) deadlineTypeEnum: Option[DeadlineTypeEnum],<br>                    @JsonScalaEnumeration(classOf[TimePeriodEnumType]) timePeriodEnum: Option[TimePeriodEnum])<br><br>case class AmountPercent(amount: AmountCurrencyAmount,<br>                         feesInclusiveInd: Option[Boolean] = Some(false),<br>                         @JsonScalaEnumeration(classOf[FeeTypeEnumType]) feeTypeEnum: Option[FeeTypeEnum],<br>                         feeValue: Option[String],<br>                         percent: Option[String], // Source side they are parsing double<br>                         taxInclusiveInd: Option[Boolean] = Some(false))<br><br>case class GuaranteesAccepted(<br>                               @JsonScalaEnumeration(classOf[PaymentCodeEnumType]) paymentTypeCodeEnum: Option[Seq[PaymentCodeEnum]])<br><br>case class DailyRate(additionalGuestAmounts: Option[Seq[AdditionalGuestAmount]],<br>                     amountDouble: Option[CurrencyAmount],<br>                     amountSingle: Option[CurrencyAmount],<br>                     netRate: Option[CurrencyAmount],<br>                     netRateDouble: Option[CurrencyAmount],<br>                     netRateSingle: Option[CurrencyAmount],<br>                     ProductnetRateSingle: Option[CurrencyAmount],<br>                     rateTypeCode: Option[String],<br>                     dailyBaseOccRate: Option[DailyRateDetails],<br>                     dailyTotalRate: DailyRateDetails)<br><br>case class DailyRateDetails(rateDetails: RateDetails)<br><br>case class RateDetails(rateAmount: RateAmount)<br><br>case class AdditionalGuestAmount(amount: Option[CurrencyAmount],<br>                                 feesIncludedInBaseInd: Option[Boolean])<br><br>case class PayStayDetails(groupBlockCode: Option[String],<br>                          ratePlanCode: String,<br>                          priceGridCode: Option[String],<br>                          inventoryTypeCode: String,<br>                          marketSegment: String,<br>                          currencyCode: String,<br>                          arrivalDate: Date,<br>                          departureDate: Date,<br>                          startDate: Date,<br>                          endDate: Date,<br>                          baseAmount: BigDecimal,<br>                          decimalPlaces: Double,<br>                          netRate: BigDecimal,<br>                          groupType: Option[String],<br>                          groupCfNumber: Option[String],<br>                          groupSequenceNumber: Int,<br>                          //todo:                       incorporate across<br>                          capsAndRestriction: CapsAndRestriction,<br>                          //                     check     res segment status<br>                          segmentStatusCode: Option[String],<br>                          roomCount: Int,<br>                          lastUpdateTime: Timestamp)<br><br><br>case class CapsAndRestriction(allowCaps: Option[String],<br>                              allowRstr: Option[String],<br>                              mirrorInvRstr: Option[String],<br>                              guarAvail: Option[String],<br>                              rateType: Option[String])<br><br>//ReservationCollationEvent to Hotel<br>case class NewNOldReservationEventEntity(reservationKey: Option[String],<br>                                         newReservation: Option[ReservationCollationEvent],<br>                                         oldReservation: Option[ReservationCollationEvent]<br>                                        )<br><br>case class NewNOldReservationEventErrorEntity(reservationKey: Option[String],<br>                                              newReservation: Option[ReservationCollationEvent],<br>                                              oldReservation: Option[ReservationCollationEvent],<br>                                              recoverableErrorFlag: Boolean)<br><br><br>case class NewNOldReservationRecordsEntity(newNOldReservations: Option[NewNOldReservationEventEntity])<br>case class NewNOldReservationRecordsErrorEntity(newNOldReservations: Option[NewNOldReservationEventErrorEntity])<br><br>case class NewNOldReservationEntity(reservationKey: Option[String],<br>                                    newReservation: Option[HotelReservationWOS3],<br>                                    oldReservation: Option[HotelReservationWOS3])<br><br>case class ReservationEventsEntity(reservationKey: Option[String],<br>                                   newReservation: Option[HotelReservationWOS3],<br>                                   oldReservation: Option[HotelReservationWOS3],<br>                                   newReservationJson: String,<br>                                   eventJson: String)<br><br>case class ReservationCollationPayload(payLoad: String,<br>                                       payLoadType: String = &quot;JSON&quot;,<br>                                       HotelReservation: Option[HotelReservationWOS3] = None) extends PayLoad<br><br>case class ReservationCollationEventPayload(val payLoad: String,<br>                                            val payLoadType: String,<br>                                            HotelReservation: Option[ReservationCollationEntity])<br><br><br>case class ReservationCollationEntity(reservationKey: Option[String], reservations: Option[OldAndNewReservation])<br><br><br>case class OldAndNewReservation(oldReservation: Option[HotelReservationWOS3],<br>                                newReservation: Option[HotelReservationWOS3])<br><br>case class ReservationCollationEvent(reservationInfo: DefaultEventInfo,<br>                                     reservationPayload: ReservationCollationPayload)<br>  extends BaseEvent[DefaultEventInfo, ReservationCollationPayload] {<br><br>  override def getEventInfo: DefaultEventInfo = reservationInfo<br><br>  override def getPayload: ReservationCollationPayload = reservationPayload<br><br>  override def eType: ru.TypeTag[DefaultEventInfo] = ru.typeTag[DefaultEventInfo]<br><br>  override def pType: ru.TypeTag[ReservationCollationPayload] = ru.typeTag[ReservationCollationPayload]<br><br>  def validateEvent(): ReservationCollationEvent = {<br>    getPayload.HotelReservation.map(row =&gt; {<br>      if (row.hotels == null || row.hotels.isEmpty) {<br>        return createErrorEvent(PropertyCodeErrorMessage)<br>      }<br>      else if (row.segments == null || row.segments.isEmpty) {<br>        return createErrorEvent(ReservationSegmentsErrorMessage)<br>      }<br>      else if (row.timespan == null || row.timespan.get.start.isEmpty) {<br>        return createErrorEvent(TimespanErrorMessage)<br>      }<br>      else if (row.products == null || row.products.forall(p =&gt; p.inventoryTypeCode == null)) {<br>        return createErrorEvent(InventoryTypeErrorMessage)<br>      }<br>    })<br>    this<br>  }<br><br>  def createErrorEvent(errorMessage: String) = {<br>    val badEntity = DefaultEventInfo(eventVersion = &quot; &quot;, eventSource = this.getEventInfo.eventSource, eventIdentifier = this.getEventInfo.eventIdentifier,<br>      recordNum = &quot; &quot;, metadata = Map(), error = ErrorTrue, retryCounter = this.getEventInfo.retryCounter + 1, destinationTopic = this.getEventInfo.destinationTopic,<br>      timeStamp = System.currentTimeMillis().toString, errorType = Seq(IrrecoverableError),<br>      message = Seq(errorMessage), warning = Seq())<br><br><br>    val reservationAggregationPayload = ReservationCollationPayload(payLoad = this.getPayload.payLoad,<br>      payLoadType = this.getPayload.payLoadType, HotelReservation = this.getPayload.HotelReservation)<br>    ReservationCollationEvent(badEntity, reservationAggregationPayload)<br><br>  }<br>}<br><br>case class ReservationLevelFields(segment: SegmentWOS3,<br>                                  groupContracts: Option[Seq[GroupContractWOS3]],<br>                                  start: String,<br>                                  end: String,<br>//                                  roomCount: Int,<br>                                  lastUpdateTime: String,<br>                                  ratePlans: Seq[RatePlanDetailsWOS3],<br>                                  productUse: ProductUseWOS3,<br>                                  dailyRate: RateChangeWOS3)<br><br>case class DerivedFields(totalGrossRevenue: Double,<br>                         totalNetRevenue: Double)<br><br>case class ReservationEvent(payLoad: String,<br>                            eventVersion: String = &quot;&quot;,<br>                            eventSource: String = &quot;reservation collation job&quot;,<br>                            eventIdentifier: String = &quot;reservation&quot;,<br>                            recordNum: String = &quot;&quot;,<br>                            timeStamp: String = &quot;&quot;,<br>                            metadata: Map[String, String] = Map(),<br>                            error: String = &quot;&quot;,<br>                            errorType: Seq[String] = Seq(),<br>                            retryCounter: Int = 0,<br>                            destinationTopic: String = &quot;crs.reservation.event&quot;,<br>                            message: Seq[String] = Seq(),<br>                            warning: Seq[String] = Seq(),<br>                            payLoadType: String = &quot;json&quot;)<br><br>/*case class ReservationCollationTriggerEvent(reservationCollationEventInfo: DefaultEventInfo,<br>                                            ReservationCollationEventPayload: ReservationCollationEventPayload)<br>  extends BaseEvent[DefaultEventInfo, ReservationCollationPayload] {<br>  override def getEventInfo: DefaultEventInfo = reservationCollationEventInfo<br><br>  override def getPayload: ReservationCollationPayload = ReservationCollationEventPayload<br><br>//  override def eType: ru.TypeTag[DefaultEventInfo] = ru.typeTag[DefaultEventInfo]<br><br>//  override def pType: ru.TypeTag[ReservationCollationPayload] = ru.typeTag[ReservationCollationPayload]<br>}*/<br><br>//modified to oldand pay as  ReservationCollationTriggerEvent to ...<br>case class ReservationEventsGroupEntity(constantGroupByKey: String,<br>                                        reservationCollationEvent: ReservationCollationEntity)<br><br>case class ReservationEventData(reservationKey: String, reservationSegments: Seq[ResEventPayStayDetails])<br><br><br>case class ResEventPayStayDetails(confirmationNumber: String,<br>                                  creationDate: String,<br>                                  propertyCode: String,<br>                                  reservationStatus: String,<br>                                  reservationArrivalDate: String,<br>                                  lastUpdateTime: String,<br>                                  groupBlockCode: String,<br>                                  ratePlan: String,<br>                                  pricePlan: Option[String] ,<br>//                                  rateType: String = &quot;&quot;,<br>                                  inventoryTypeCode: String,<br>                                  marketSegment: String,<br>                                  currencyCode: String,<br>                                  segmentArrivalDate: String,<br>                                  segmentDepartureDate: String,<br>                                  groupType: String,<br>                                  groupCfNumber: String,<br>                                  segmentLOS: String,<br>                                  segmentLOSOffset: Int,<br>                                  totalLOS: Int,<br>                                  compRoomCount: Int = 0,<br>//                                  allowCaps: String = &quot;&quot;,<br>//                                  allowRstr: String = &quot;&quot;,<br>//                                  mirrorInvRstr: String = &quot;&quot;,<br>                                  guarAvail: String = &quot;&quot;,<br>                                  stayRoomCount: Int = 0,<br>                                  stayGrossRevenue: Double = 0.0,<br>                                  stayNetRevenue: Double = 0.0,<br>                                  totalStayGrossRevenue: Double = 0.0,<br>                                  totalStayNetRevenue: Double = 0.0,<br>                                  cancelledStayRoomCount: Int = 0,<br>                                  cancelledStayRevenue: Double = 0.0,<br>                                  payRoomCount: Int = 0,<br>                                  payGrossRevenue: Double = 0.0,<br>                                  payNetRevenue: Double = 0.0,<br>                                  totalPayGrossRevenue: Double = 0.0,<br>                                  totalPayNetRevenue: Double = 0.0,<br>                                  cancelledPayRoomCount: Int = 0,<br>                                  cancelledPayRevenue: Double = 0.0,<br>                                  noShowRooms: Option[Int] = Some(0),<br>                                  noShowRevenue: Option[Double] = Some(0.0),<br>                                  noShowPayRooms: Option[Int] = Some(0),<br>                                  noShowPayRevenue: Option[Double] = Some(0.0),<br>                                  sameDayCheckIn: Option[Int] = Some(0),<br>                                  earlyCheckout: Option[Int] = Some(0),<br>                                  unexpectedStay: Option[Int] = Some(0),<br>                                  changeInLOS: Option[Int] = Some(0))<br><br>case class ImplicitCouchDetails(bucketName: String,<br>                                n1QlQuery: String)<br><br><br>case class RateProgramDocument(documentId: String,<br>                               capsAndRstr: CapsAndRstr)<br><br><br>case class ViewQueryResult(rateOffer: RateOffer, capsAndRstr: CapsAndRstr)<br><br>case class RateOffer(rateType: Option[String])<br><br>case class CapsAndRstr(guarAvail: Option[Boolean])<br><br>//to vheck whether its a hotel or custom reservation<br>/*case class OldAndNewReservation(oldReservation: Option[HotelReservation],<br>                                newReservation: Option[CustomReservation])*/<br>/*<br><br>case class CustomReservation(hotelReservation: HotelReservation)<br><br><br>case class ReservationCollationEntity(reservationKey: Option[String], reservations: Option[OldAndNewReservation])*/<br>case class ReservationEventLevelFields(confirmationNumber: String,<br>                                       creationDateTime: String,<br>                                       reservationArrivalDate: String,<br>                                       reservationDepartureDate: String,<br>                                       reservationStatus: String,<br>                                       propertyCode: String,<br>                                       lastUpdateTime: String,<br>                                       totalGrossRevenue: Double,<br>                                       totalNetRevenue: Double,<br>//                                       inventoryType: String,<br>//                                       roomCount: Int,<br>                                       totalLOS: Int,<br>                                       brandCode: String)<br><br>case class ConditionalFlags(noShowRooms: Option[Int] = Some(0),<br>                            noShowRevenue: Option[Double] = Some(0.0),<br>                            noShowPayRooms: Option[Int] = Some(0),<br>                            noShowPayRevenue: Option[Double] = Some(0.0),<br>                            sameDayCheckIn: Option[Int] = Some(0),<br>                            earlyCheckout: Option[Int] = Some(0),<br>                            unexpectedStay: Option[Int] = Some(0),<br>                            var changeInLOS: Option[Int] = Some(0))<br><br>case class EEHData(<br>                    timeStamp: String,<br>                    metadata: Option[Map[String, Any]],<br>                    retryCounter: Int,<br>                    payLoad: String,<br>                    eventVersion: String,<br>                    eventSource: String,<br>                    destinationTopic: String,<br>                    eventIdentifier: String,<br>                    payLoadType: String,<br>                    recordNum: String,<br>                    error: Option[String],<br>                    errorType: Option[Seq[Any]],<br>                    message: Option[Seq[Any]],<br>                    warning: Option[Seq[Any]]<br>                  )<br><br><br>case class ReservationKafkaEvent(timeStamp: String = DateTimeFormatter.ofPattern(DATE_TIME_FORMAT).format(LocalDateTime.now()),<br>                                 metadata: Map[String, String] = Map(),<br>                                 retryCounter: Int = 0,<br>                                 payLoad: String,<br>                                 eventVersion: String = EmptyString,<br>                                 eventSource: String = EventSourceValue,<br>                                 destinationTopic: String,<br>                                 eventIdentifier: String = EventIdentifierValue,<br>                                 payLoadType: String = PayLoadTypeValue,<br>                                 recordNum: String = EmptyString,<br>                                 error: String = EmptyString,<br>                                 errorType: Seq[String] = Seq(),<br>                                 message: Seq[String] = Seq(),<br>                                 warning: Seq[String] = Seq()) extends StateInput<br><br>// Rate-plan details<br>case class RatesBucketData (<br>                             ratePlanDetails:RatePlanDetailsDB<br>                           )<br>case class RatePlanDetailsDB(<br>                              marketType: Option[MarketType])<br>exceptions<br>helper<br><br>documentdbSinkHelper<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.marriott.elevate.spark.framework.config.MessageBrokerDetails<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}<br><br>import scala.util.{Failure, Success, Try}<br><br>trait DocumentDBSinkHelper extends LazyLogging with Serializable {<br>  /**<br>    * Method that pushes the record to kafka topic<br>    *<br>    * @param record - record that needs to be pushed to kafka<br>    * @param config - kafka configuration details<br>    */<br>  def pushRecordToKafka(record: String, config: MessageBrokerDetails, topicName: String,kafkaProducer:KafkaProducer[String, String]): Unit = {<br><br>    Try {<br>      logger.info(&quot;Kafka ::: Pushing records to Kafka&quot;)<br>      val data = new ProducerRecord[String, String](topicName, record)<br>      kafkaProducer.send(data)<br><br>    }<br>    match {<br>      case Success(_) =&gt; logger.info(&quot;Record pushed to Kafka topic &quot; + config.destinationtopic + &quot; successfully!&quot;)<br>      case Failure(error) =&gt;<br>        logger.error(&quot;Error while pushing the record to Kafka : &quot; + error.getMessage)<br>    }<br>  }<br>}<br><br><br>ReservationCollationHelper<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.fasterxml.jackson.databind.ObjectMapper<br>import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants._<br>import com.marriott.elevate.spark.framework.entity.DefaultEventInfo<br>import com.marriott.elevate.spark.framework.utils.JsonUtil<br>import com.marriott.elevate.spark.framework.utils.JsonUtil.toJson<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationStatusEnum.`Committed`<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion.exceptions.CustomException<br>import com.mongodb.client.model.Filters<br>import org.mongodb.scala.model.Filters.or<br>//import com.marriott.mgpapp.spark.reservation.ingestion.helper.CustomExceptionHandler.errorHandler<br>//import com.marriott.rm.platform.exception.event.BaseException.toBaseException<br>import com.marriott.utility.CustomLogger<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.commons.lang3.exception.ExceptionUtils<br>import org.apache.spark.SparkConf<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.expressions.UserDefinedFunction<br>import org.apache.spark.sql.functions._<br>import org.mongodb.scala.bson.Document<br><br>import java.sql.Date<br>import java.time.temporal.ChronoUnit<br>import java.time.temporal.ChronoUnit.DAYS<br>import java.time.{LocalDate, LocalDateTime}<br>import scala.Console.println<br>import scala.math.Numeric.IntIsIntegral<br>import scala.util.{Failure, Success, Try}<br><br>trait ReservationCollationHelper extends LazyLogging with Serializable with ReservationIngestionHelper {<br><br>  //  implicit val jobConfig: JobConfigurationDetails = jobConfigDetails<br>  override implicit val jobIdentifier: String = ReservationIngestion<br>  override implicit val jobName: String = Reservations<br><br>  override val log: CustomLogger = new CustomLogger(this.getClass.getName)<br><br>  def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups(getLatestReservationRecord)<br>    //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>    //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty<br>      )<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br>    //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>    //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    /**<br>     * compare and modify the pay and stay details for reservation record and<br>     * add the same to each and every segment in the record<br>     */<br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>    //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>    //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>    (newNOldRecordsDS, recoverableDocDBDF)<br>  }<br><br>  def getReservationEvents_Map(<br>                                reservationDS: NewNOldReservationEventErrorEntity<br>                              ): ReservationEventsEntity = {<br><br>    log.info(&quot;Starting the processing of comparison and modification of pay &amp; stay details for reservation records &quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val newReservation = reservationDS.newReservation match {<br>      case Some(newRes) =&gt; newRes.reservationPayload.HotelReservation<br>      case None =&gt; None<br>    }<br><br>    val result = ReservationEventsEntity(reservationDS.reservationKey,<br>      newReservation,<br>      reservationDS.oldReservation match {<br>        case Some(oldRes) =&gt; oldRes.reservationPayload.HotelReservation<br>        case None =&gt; None<br>      },<br>      toJson(Map(&quot;HotelReservation&quot; -&gt; newReservation))<br>      , &quot;None&quot;)<br>    //println(&quot;5 Aug - Data while performing mapping in collation step - &quot; + result)<br>    result<br>  }<br><br>  /**<br>   * Function to get the reservation key out of reservation record<br>   *<br>   * @param reservationRecord : reservation record along with event info<br>   * @return : the key for the reservation record which is combination of<br>   *         reservationCode and creationDateTime delimited by underscore(_) */<br>  def getReservationKey(reservationRecord: ReservationCollationEvent, jobConfig: JobConfigurationDetails): Option[String] = {<br><br>    log.info(&quot;Fetching the Reservation key value - getReservationKey()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    println(&quot;error check:&quot; + reservationRecord.reservationPayload.HotelReservation.get.confirmationNumber)<br>    println(&quot;error check for date:&quot; + reservationRecord.reservationPayload.HotelReservation.get.dateTimeStamp.get.create)<br>    log.info(&quot; - Value of Reservation Key : &quot; +<br>      (reservationRecord.reservationPayload.HotelReservation.get.confirmationNumber.value<br>        + &quot;_&quot; + reservationRecord.reservationPayload.HotelReservation.get.dateTimeStamp.get.create.get.substring(0, 10)),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    Some(reservationRecord.reservationPayload.HotelReservation.get.confirmationNumber.value + &quot;_&quot; +<br>      reservationRecord.reservationPayload.HotelReservation.get.dateTimeStamp.get.create.get.substring(0, 10))<br>  }<br><br><br>  /**<br>   * Function to get Pay and stay details for the reservation record<br>   *<br>   * @param reservationLevelFields : reservation level fields to be updated as part<br>   *                               of pay and stay details<br>   * @return : case class with all the required pay stay details<br>   */<br>  def getPayStayDetails(reservationLevelFields: ReservationLevelFields, hotel: HotelWOS3): PayStayDetailsWOS3 = {<br><br>    log.info(&quot;Fetching the pay &amp; stay details for the respective reservation record - getPayStayDetails()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>//    val (baseAmount, netRate) = if (reservationLevelFields.productUse.productRates.dailyRates.<br>//      head.dailyRate.dailyBaseOccRate.isDefined)<br>//      (reservationLevelFields.segment.offer.get.productUses.head.productRates.dailyRates.<br>//        head.dailyRate.dailyBaseOccRate.get.rateDetails.rateAmount.bsAmt.get.value, reservationLevelFields.segment.offer.get.productUses.head.productRates.dailyRates.<br>//        head.dailyRate.dailyBaseOccRate.get.rateDetails.rateAmount.amtBfTx.get.value)<br>//    else (reservationLevelFields.segment.offer.get.productUses.head.productRates.dailyRates.<br>//      head.dailyRate.dailyTotalRate.rateDetails.rateAmount.bsAmt.get.value, reservationLevelFields.segment.offer.get.productUses.head.productRates.dailyRates.<br>//      head.dailyRate.dailyTotalRate.rateDetails.rateAmount.amtBfTx.get.value)<br><br>    val (baseAmount, netRate) = reservationLevelFields.dailyRate.dailyRate.dailyBaseOccRate.map { baseOccRate =&gt;<br>          (<br>            baseOccRate.rateDetails.rateAmount.amtAfTx.map(_.value).getOrElse(BigDecimal(0)),<br>            baseOccRate.rateDetails.rateAmount.amtBfTx.map(_.value).getOrElse(BigDecimal(0))<br>          )<br>        }.getOrElse {<br>          (<br>            reservationLevelFields.dailyRate.dailyRate.dailyTotalRate.rateDetails.rateAmount.amtAfTx.map(_.value).getOrElse(BigDecimal(0)),<br>            reservationLevelFields.dailyRate.dailyRate.dailyTotalRate.rateDetails.rateAmount.amtBfTx.map(_.value).getOrElse(BigDecimal(0))<br>          )<br>        }<br><br>    val ratePlanMap = reservationLevelFields.ratePlans.map(ratePlan =&gt;<br>      ratePlan.code -&gt; ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;)<br>    ).toMap<br><br>    val groupType = reservationLevelFields.groupContracts.flatMap { groupContracts =&gt;<br>      groupContracts.find(_.code.contains(reservationLevelFields.productUse.groupBlockCode.getOrElse(&quot;&quot;)))<br>        .flatMap(_.groupType)<br>    }<br><br>    val groupCfNumber = reservationLevelFields.groupContracts.flatMap { groupContracts =&gt;<br>      groupContracts.collectFirst {<br>        case groupContract if groupContract.externalIds.exists(_.exists(item =&gt;<br>          item.`type`.map(_.trim.toLowerCase).contains(&quot;sequence id&quot;) &amp;&amp; item.id.isDefined<br>        )) =&gt;<br>          groupContract.externalIds.flatMap(_.find(item =&gt; item.`type`.map(_.trim.toLowerCase).contains(&quot;sequence id&quot;) &amp;&amp; item.id.isDefined)).get.id.get<br>      }.orElse(groupContracts.headOption.map(_.confirmationNumber.value))<br>    }<br><br>    val priceGridMap = reservationLevelFields.ratePlans.map { ratePlan =&gt;<br>      val priceGridCode = ratePlan.packageProducts.flatMap(x =&gt; x.headOption.map(y =&gt; y.priceGridCode.getOrElse(&quot;&quot;))).getOrElse(&quot;&quot;)<br>      ratePlan.code -&gt; priceGridCode<br>    }.toMap<br><br>    val ratePlanCode = reservationLevelFields.productUse.ratePlanCode.getOrElse(&quot;&quot;)<br><br>    val result = PayStayDetailsWOS3(<br>      reservationLevelFields.productUse.groupBlockCode,<br>      ratePlanCode,<br>      Some(priceGridMap.getOrElse(ratePlanCode, &quot;&quot;)),<br>      reservationLevelFields.productUse.inventoryTypeCode.getOrElse(&quot;&quot;),<br>      ratePlanMap.getOrElse(ratePlanCode, &quot;&quot;),<br>      hotel.propertyCurrency.getOrElse(&quot;&quot;),<br>      reservationLevelFields.start,<br>      reservationLevelFields.end,<br>      reservationLevelFields.dailyRate.timespan.map(_.start).getOrElse(&quot;&quot;),<br>      reservationLevelFields.dailyRate.timespan.map(_.end).getOrElse(&quot;&quot;),<br>      baseAmount,<br>      reservationLevelFields.segment.totalSegment.flatMap(_.numberOfDecimals).map(_.toDouble).getOrElse(0.0),<br>      netRate,<br>      groupType,<br>      groupCfNumber,<br>      CapsAndRestrictionWOS3(None),<br>      reservationLevelFields.segment.actionStatusEnum.map(_.toString),<br>      reservationLevelFields.productUse.quantity,<br>      reservationLevelFields.lastUpdateTime<br>    )<br>    //println(&quot;5 Aug - Collation | getPayStayDetails - &quot; + result)<br>    result<br>  }<br><br>  /**<br>   * Function to get reservation records modified with pay and stay details<br>   *<br>   * @param newReservation : new reservation record coming from kafka<br>   * @param oldReservation : old reservation record coming from couchbase<br>   * @return : the function returns sequence of segments with pay and stay details<br>   */<br>  def getReservationWithPayNStayDetails(newReservation: ReservationCollationEvent,<br>                                        oldReservation: Option[ReservationCollationEvent]): (ReservationCollationEvent, Option[ReservationCollationEvent]) = {<br><br>    log.info(&quot;Fetching records modified with pay and stay details - getReservationWithPayNStayDetails()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val newDerivedFields: DerivedFields = getRequiredFieldValues(newReservation)<br>    //println(&quot;5 Aug - Collation | getReservationWithPayNStayDetails | new res derived fields - &quot; + newDerivedFields)<br><br>    val newResLos = getLOS(newReservation.reservationPayload.HotelReservation.get<br>      .timespan.get.start,<br>      newReservation.reservationPayload.HotelReservation.get<br>        .timespan.get.end)<br>    //println(&quot;5 Aug - Collation | getReservationWithPayNStayDetails | los - &quot; + newResLos)<br><br>    val (oldResWithPayStayDetails, oldTotalGrossRevenue, oldTotalNetRevenue, oldInventoryType, oldTotalLos) = oldReservation match {<br><br>      case Some(oldResRecord) =&gt;<br>        val segment = oldResRecord.reservationPayload.HotelReservation.get.segments.head<br><br>        val oldArrivalDate = oldResRecord.reservationPayload.HotelReservation<br>          .get.timespan.get.start<br><br>        val oldDepartureDate = oldResRecord.reservationPayload.HotelReservation<br>          .get.timespan.get.end<br><br>//        val oldRoomCount = oldResRecord.reservationPayload.HotelReservation<br>//          .get.segments.head.offer.get.productUses.head.quantity<br><br>        val oldLastUpdateTime = oldResRecord.reservationPayload.HotelReservation<br>          .get.dateTimeStamp.get.lastModify.get<br><br>        val ratePlans = oldResRecord.reservationPayload.HotelReservation.get.ratePlans<br>        val groupContracts = oldResRecord.reservationPayload.HotelReservation.get.groupContracts<br>//        val inventoryTypeCode = oldResRecord.reservationPayload.HotelReservation.get.products.head.inventoryTypeCode<br><br>        val oldDerivedFields = getRequiredFieldValues(oldResRecord)<br><br>        val (oldNonCompRoomSegmentsAndGroupContracts, oldCompRoomSegmentsAndGroupContracts) = bifurcateNonCompNCompProductUses(oldResRecord)<br>        val oldInventoryTypes = oldNonCompRoomSegmentsAndGroupContracts.map(_.inventoryTypeCode.getOrElse(&quot;&quot;)) ++ oldCompRoomSegmentsAndGroupContracts.map(_.inventoryTypeCode.getOrElse(&quot;&quot;))<br><br>//        val nonCompRoomsSegPayDetailsSeq = oldNonCompRoomSegmentsAndGroupContracts<br>//          .map(segmentAndGroupContract =&gt; {<br>//            val oldReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//              segmentAndGroupContract._2,<br>//              oldArrivalDate,<br>//              oldDepartureDate,<br>////              oldRoomCount,<br>//              oldLastUpdateTime.get,<br>//              ratePlans.head,<br>//              inventoryTypeCode)<br>//<br>//            getPayStayDetails(oldReservationLevelFields, oldReservation.get.reservationPayload.HotelReservation.get.hotels.head)<br>//          })<br>val oldNonCompPayStayDetails = oldNonCompRoomSegmentsAndGroupContracts.map(nonCompProductUse =&gt; {<br>  val nonCompDailyRates = nonCompProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br><br>    val oldReservationLevelFields = ReservationLevelFields(segment,<br>      groupContracts,<br>      oldArrivalDate, oldDepartureDate, oldLastUpdateTime,<br>      ratePlans, nonCompProductUse, dailyRate)<br>    val payStayDetails = getPayStayDetails(oldReservationLevelFields, oldReservation.get.reservationPayload.HotelReservation.get.hotels.head)<br>    if (dailyRate.payDetails.isEmpty &amp;&amp; dailyRate.stayDetails.isEmpty) {<br>      dailyRate.copy(payDetails = Some(payStayDetails),<br>        stayDetails = Some(payStayDetails))<br>    } else {<br>      dailyRate<br>    }<br>  })<br>  nonCompProductUse.copy(productRates = nonCompProductUse.productRates.copy(dailyRates = nonCompDailyRates))<br>})<br>        val oldCompStayDetails = oldCompRoomSegmentsAndGroupContracts.map(compProductUse =&gt; {<br>          val compDailyRates = compProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>          val oldReservationLevelFields = ReservationLevelFields(segment,<br>            groupContracts,<br>            oldArrivalDate, oldDepartureDate, oldLastUpdateTime,<br>            ratePlans, compProductUse, dailyRate)<br>          val stayDetails = getPayStayDetails(oldReservationLevelFields, oldReservation.get.reservationPayload.HotelReservation.get.hotels.head)<br>            dailyRate.copy(stayDetails = Some(stayDetails))<br>          })<br>          compProductUse.copy(productRates = compProductUse.productRates.copy(dailyRates = compDailyRates))<br>        })<br><br>//        val compRoomSegWithStayDetails = oldCompRoomSegmentsAndGroupContracts<br>//          .map(segmentAndGroupContract =&gt; {<br>//            val oldReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//              segmentAndGroupContract._2,<br>//              oldArrivalDate,<br>//              oldDepartureDate,<br>////              oldRoomCount,<br>//              oldLastUpdateTime.get,<br>//              ratePlans.head,<br>//              inventoryTypeCode)<br>//<br>//            segmentAndGroupContract._1.copy(stayDetails = Some(getPayStayDetails(oldReservationLevelFields, oldReservation.get.reservationPayload.HotelReservation.get.hotels.head)))<br>//          })<br><br>//        val oldResNonCompSegWithPayStayDetails = oldNonCompRoomSegmentsAndGroupContracts<br>//          .map(segmentAndGroupContract =&gt; {<br>//<br>//            log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of segment.payDetails =&gt; &quot; +<br>//              segmentAndGroupContract._1.payDetails.toString, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//            log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of segment.stayDetails =&gt; &quot; +<br>//              segmentAndGroupContract._1.stayDetails.toString, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//<br>//            if (segmentAndGroupContract._1.payDetails.isEmpty &amp;&amp; segmentAndGroupContract._1.stayDetails.isEmpty) {<br>//              val oldReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//                segmentAndGroupContract._2,<br>//                oldArrivalDate,<br>//                oldDepartureDate,<br>////                oldRoomCount,<br>//                oldLastUpdateTime.get,<br>//                ratePlans.head,<br>//                inventoryTypeCode)<br>//              segmentAndGroupContract._1.copy(<br>//                payDetails = Some(nonCompRoomsSegPayDetailsSeq),<br>//<br>//                stayDetails = Some(getPayStayDetails(oldReservationLevelFields, oldReservation.get.reservationPayload.HotelReservation.get.hotels.head)))<br>//            } else<br>//              segmentAndGroupContract._1<br>//          })<br><br>        val oldResWithPayStayDetails = oldResRecord<br>          .copy(reservationPayload = oldResRecord<br>            .reservationPayload.copy(HotelReservation = Some((oldResRecord<br>            .reservationPayload.HotelReservation.get<br>              .copy(totalGrossRevenue = Some(oldDerivedFields.totalGrossRevenue),<br>                totalNetRevenue = Some(oldDerivedFields.totalNetRevenue),<br>                segments = Seq(segment.copy(offer = Some(segment.offer.get<br>                  .copy(productUses = oldNonCompPayStayDetails ++ oldCompStayDetails)))))))))<br><br><br>        val (oldInventoryType, oldTotalLos) =<br>          (oldInventoryTypes,<br>            getLOS(oldResWithPayStayDetails.reservationPayload.HotelReservation.get.timespan.get.start,<br>              oldResWithPayStayDetails.reservationPayload.HotelReservation.get.timespan.get.end))<br><br>        log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of oldInventoryType =&gt; &quot; + oldInventoryType,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of PayStayDetail =&gt; &quot; + oldTotalLos,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>        (Some(oldResWithPayStayDetails), oldDerivedFields.totalGrossRevenue,<br>          oldDerivedFields.totalNetRevenue, oldInventoryType, oldTotalLos)<br><br>      case None =&gt; (None, 0.00, 0.00, Seq.empty[String], 0)<br>    }<br><br>    val oldArrivalDate = if (oldResWithPayStayDetails.nonEmpty &amp;&amp; oldResWithPayStayDetails.head.reservationPayload.HotelReservation.get.timespan.get.start.nonEmpty) {<br>      oldResWithPayStayDetails.head.reservationPayload.HotelReservation.get.timespan.get.start<br>    } else {<br>      None<br>    }<br>    val newArrivalDate = newReservation.reservationPayload.HotelReservation.get.timespan.get.start.head<br><br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of oldArrivalDate =&gt; &quot; + oldArrivalDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of newArrivalDate =&gt; &quot; + newArrivalDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of oldTotalGrossRevenue =&gt; &quot; + oldTotalGrossRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of totalGrossRevenue =&gt; &quot; + newDerivedFields.totalGrossRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of oldTotalNetRevenue =&gt; &quot; + oldTotalNetRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of totalNetRevenue =&gt; &quot; + newDerivedFields.totalNetRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of newResLos =&gt; &quot; + newResLos,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getReservationWithPayNStayDetails() | Value of oldTotalLos =&gt; &quot; + oldTotalLos,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //check for upgrade and upsell scenario<br>    if (newDerivedFields.totalGrossRevenue == oldTotalGrossRevenue &amp;&amp;<br>      newDerivedFields.totalNetRevenue == oldTotalNetRevenue &amp;&amp;<br>      oldInventoryType.nonEmpty)<br>      if (newResLos == oldTotalLos)<br>        if (oldArrivalDate == None || oldArrivalDate == Some(newArrivalDate)) {<br>          //Upgrade or change in any of the key fields -- carry forward payDetails from older reservation<br>          (getModifiedReservationRecord(newReservation, oldResWithPayStayDetails), oldResWithPayStayDetails)<br>        }<br>        else<br>        //Upsell scenario when change in arrivaldate in this block<br>          (getModifiedReservationRecord(newReservation), oldResWithPayStayDetails)<br>      else<br>      //Upsell scenario<br>        (getModifiedReservationRecord(newReservation), oldResWithPayStayDetails)<br>    else<br>    //Upsell scenario<br>      (getModifiedReservationRecord(newReservation), oldResWithPayStayDetails)<br>  }<br><br>  /**<br>   * Get LOS between the end date and start date<br>   *<br>   * @param startDate<br>   * @param endDate<br>   * @return<br>   */<br>//  override def getLOS(startDate: Date, endDate: Date): Int = {<br>//<br>//    log.info(&quot;Get the length of stay between the start &amp; end date - getLOS()&quot;,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//<br>//    val startDt = LocalDate.parse(startDate.toString)<br>//    val endDt = LocalDate.parse(endDate.toString)<br>//<br>//    log.info(&quot; - getLOS() - Value of LOS =&gt; &quot; + DAYS.between(startDt, endDt).toInt.toString,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//<br>//    DAYS.between(startDt, endDt).toInt<br>//  }<br><br>  /**<br>   * Function to get reservation records modified with pay and stay details<br>   *<br>   * @param newReservation<br>   * @param oldResWithPayStayDetails<br>   * @param derivedFields : set of confirmationNumber, parentConfirmationNumber<br>   *                      * totalGrossRevenue and totalNetRevenue fields<br>   * @return : reservation record with pay and stay details<br>   */<br>  def getModifiedReservationRecord(newReservation: ReservationCollationEvent,<br>                                   oldResWithPayStayDetails: Option[ReservationCollationEvent] = None)<br>                                  (implicit derivedFields: DerivedFields): ReservationCollationEvent = {<br><br>    log.info(&quot;Fetching  reservation records modified with pay and stay details - getModifiedReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    println(&quot;oldResWithPayStayDetails :&quot; ,oldResWithPayStayDetails)<br><br>    val segment = newReservation.reservationPayload.HotelReservation.get.segments.head<br><br>    val arrivalDate = newReservation.reservationPayload.HotelReservation.get.timespan.get.start<br><br>    val departureDate = newReservation.reservationPayload.HotelReservation.get.timespan.get.end<br><br>//    val newRoomCount = newReservation.reservationPayload.HotelReservation.get<br>//      .segments.head.offer.get.productUses.head.quantity<br><br>    val newLastUpdateTime = newReservation.reservationPayload.HotelReservation.get<br>      .dateTimeStamp.get.lastModify.get<br><br>    val ratePlans = newReservation.reservationPayload.HotelReservation.get.ratePlans<br><br>    val groupContracts = newReservation.reservationPayload.HotelReservation.get.groupContracts<br>//    val inventoryTypeCode = newReservation.reservationPayload.HotelReservation.get.products.head.inventoryTypeCode<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res arrival date - &quot; + arrivalDate)<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res departure date - &quot; + departureDate)<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res roomcount - &quot; + newRoomCount)<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res lastmodify - &quot; + newLastUpdateTime)<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res rateplan - &quot; + ratePlans)<br>    //println(&quot;5 Aug - Collation | getModifiedReservationRecord | new res inventoryTypecode - &quot; + roomPoolCode)<br><br>    /*<br>     * copy the confirmationNumber, parentConfirmationNubmer, totalGrossRevenue, totalNetRevenue and<br>     * to pay stay details to the new reservation record<br>     */<br>    oldResWithPayStayDetails match {<br>      case Some(oldReservation) =&gt;<br><br>        val oldNonCompRoomSegmentsAndGroupContracts = bifurcateNonCompNCompProductUses(oldReservation)._1<br><br>        val (newNonCompRoomSegments, newCompRoomSegments) = bifurcateNonCompNCompProductUses(newReservation)<br><br>        val oldPayDetailsSeq = if (oldNonCompRoomSegmentsAndGroupContracts.isEmpty) None<br>        else Some(oldNonCompRoomSegmentsAndGroupContracts.flatMap(productUse =&gt; {<br>          productUse.productRates.dailyRates.flatMap(dailyRate =&gt; {<br>            dailyRate.payDetails<br>          })<br>        }))<br><br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | old res non comp - &quot; + oldNonCompRoomSegmentsAndGroupContracts)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | new res non comp - &quot; + newNonCompRoomSegments)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | new res comp - &quot; + newCompRoomSegments)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | old res pay details - &quot; + oldPayDetailsSeq)<br><br>//        val newNonCompRoomSegWithPayStayDetails = newNonCompRoomSegments<br>//          .map(newSegmentAndGroupContract =&gt; {<br>//            val payDetails = if (newSegmentAndGroupContract._1.payDetails.isDefined) {<br>//              log.info(&quot; - Function - getModifiedReservationRecord() | Value of oldPayDetailsSeq =&gt; &quot; +<br>//                oldPayDetailsSeq.isDefined, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//              log.info(&quot; - Function - getModifiedReservationRecord() | Value of newSegment.lastUpdateTime =&gt; &quot; +<br>//                newSegmentAndGroupContract._1.payDetails.get.head.lastUpdateTime, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//              log.info(&quot; - Function - getModifiedReservationRecord() | Value of oldPayDetailsSeq.lastUpdateTime =&gt; &quot; +<br>//                oldPayDetailsSeq.get.head.lastUpdateTime, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//<br>//              if (oldPayDetailsSeq.isDefined &amp;&amp; convertStringToTimestamp(newSegmentAndGroupContract._1.payDetails.get.head.lastUpdateTime)<br>//                .before(convertStringToTimestamp(oldPayDetailsSeq.get.head.lastUpdateTime)))<br>//                newSegmentAndGroupContract._1.payDetails<br>//              else oldPayDetailsSeq<br>//            } else oldPayDetailsSeq<br>//<br>//            val newReservationLevelFields = ReservationLevelFields(newSegmentAndGroupContract._1, newSegmentAndGroupContract._2, arrivalDate, departureDate, newRoomCount,<br>//              newLastUpdateTime, ratePlans.head, inventoryTypeCode)<br>//<br>//            val newStayDetails = getPayStayDetails(newReservationLevelFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>//            //logger.info(&quot;Collation::Reservation Modification - Upgrade scenario&quot;)<br>//            newSegmentAndGroupContract._1.copy(payDetails = payDetails, stayDetails = Some(newStayDetails))<br>//          })<br>        val newNonCompPayStay = newNonCompRoomSegments.map(nonCompProductUse =&gt; {<br>          val nonCompDailyRates = nonCompProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>            val oldPayDetail = if (oldPayDetailsSeq.isDefined) {<br>              oldPayDetailsSeq.get.filter(paydetails =&gt; ((paydetails.startDate == dailyRate.payDetails.get.startDate) &amp;&amp; (paydetails.endDate == dailyRate.payDetails.get.endDate)))<br>            } else Seq.empty[PayStayDetailsWOS3]<br>            val payDetails = if (dailyRate.payDetails.isDefined) {<br>              if (oldPayDetail.nonEmpty &amp;&amp; convertStringToTimestamp(dailyRate.payDetails.get.lastUpdateTime)<br>                .before(convertStringToTimestamp(oldPayDetail.head.lastUpdateTime))) {<br>                dailyRate.payDetails.get<br>              } else {<br>                oldPayDetail.head<br>              }<br>            } else {<br>              oldPayDetail.head<br>            }<br>            val newResFields = ReservationLevelFields(segment, groupContracts, arrivalDate, departureDate, newLastUpdateTime, ratePlans, nonCompProductUse, dailyRate)<br>            val stayDetails = getPayStayDetails(newResFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>            dailyRate.copy(payDetails = Some(payDetails),<br>              stayDetails = Some(stayDetails))<br>          })<br>          nonCompProductUse.copy(productRates = nonCompProductUse.productRates.copy(dailyRates = nonCompDailyRates))<br>        })<br><br>        val newCompStay = newCompRoomSegments.map(compProductUse =&gt; {<br>          val compDailyRates = compProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>          val newResFields = ReservationLevelFields(segment, groupContracts, arrivalDate, departureDate, newLastUpdateTime, ratePlans, compProductUse, dailyRate)<br>          val stayDetails = getPayStayDetails(newResFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>            dailyRate.copy(stayDetails = Some(stayDetails))<br>          })<br>          compProductUse.copy(productRates = compProductUse.productRates.copy(dailyRates = compDailyRates))<br>        })<br><br>//        val compRoomSegWithStayDetails = newCompRoomSegments<br>//          .map(newSegmentAndGroupContract =&gt; {<br>//            val newReservationLevelFields = ReservationLevelFields(newSegmentAndGroupContract._1,<br>//              newSegmentAndGroupContract._2, arrivalDate, departureDate, newRoomCount,<br>//              newLastUpdateTime, ratePlans.head, inventoryTypeCode)<br>//            newSegmentAndGroupContract._1.copy(stayDetails = Some(getPayStayDetails(newReservationLevelFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)))<br>//          })<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | new non comp + paystay details - &quot; + newNonCompRoomSegWithPayStayDetails)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res present | new comp + stay details - &quot; + compRoomSegWithStayDetails)<br><br>        newReservation<br>          .copy(reservationPayload = newReservation.reservationPayload<br>            .copy(HotelReservation = Some(newReservation<br>              .reservationPayload.HotelReservation.get<br>              .copy(totalGrossRevenue = Some(derivedFields.totalGrossRevenue),<br>                totalNetRevenue = Some(derivedFields.totalNetRevenue),<br>                segments = Seq(segment.copy(offer = Some(segment.offer.get<br>                  .copy(productUses = newNonCompPayStay ++ newCompStay))))))))<br><br>      case None =&gt;<br><br>        val (nonCompRoomSegments, compRoomSegments) = bifurcateNonCompNCompProductUses(newReservation)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res not present | new res non comp - &quot; + nonCompRoomSegments)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res not present | new res comp - &quot; + compRoomSegments)<br><br>//        val newPayDetails = if (nonCompRoomSegments.isEmpty) Seq()<br>//        else if (nonCompRoomSegments.head._1.payDetails.isEmpty)<br>//          nonCompRoomSegments<br>//            .map(segmentAndGroupContract =&gt; {<br>//              val newReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//                segmentAndGroupContract._2,<br>//                arrivalDate, departureDate, newRoomCount, newLastUpdateTime,<br>//                ratePlans.head, inventoryTypeCode)<br>//              getPayStayDetails(newReservationLevelFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>//            })<br>//        else<br>//          nonCompRoomSegments.head._1.payDetails.get<br><br>        val newPayStayNonComp = if (nonCompRoomSegments.isEmpty) {<br>          Seq.empty[ProductUseWOS3]<br>        } else {<br>          nonCompRoomSegments.map(nonCompProductUse =&gt; {<br>            val nonCompDailyRates = nonCompProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br><br><br>            val newResFields = ReservationLevelFields(segment, groupContracts, arrivalDate, departureDate, newLastUpdateTime, ratePlans, nonCompProductUse, dailyRate)<br>            val payDetails = if (dailyRate.payDetails.isDefined) {<br>              dailyRate.payDetails.get<br>            } else {<br>              getPayStayDetails(newResFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>            }<br>            val stayDetails = getPayStayDetails(newResFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>              dailyRate.copy(payDetails = Some(payDetails),<br>              stayDetails = Some(stayDetails))<br>            })<br>            nonCompProductUse.copy(productRates = nonCompProductUse.productRates.copy(dailyRates = nonCompDailyRates))<br>          })<br>        }<br><br>        val newCompStay = compRoomSegments.map(compProductUse =&gt; {<br>          val compDailyRates = compProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>          val newResFields = ReservationLevelFields(segment,<br>            groupContracts,<br>            arrivalDate, departureDate, newLastUpdateTime,<br>            ratePlans, compProductUse, dailyRate)<br>          val stayDetails = getPayStayDetails(newResFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)<br>            dailyRate.copy(stayDetails = Some(stayDetails))<br>          })<br>          compProductUse.copy(productRates = compProductUse.productRates.copy(dailyRates = compDailyRates))<br>        })<br><br>//        val compRoomSegWithStayDetails = compRoomSegments<br>//          .map(newSegmentAndGroupContract =&gt; {<br>//            val newReservationLevelFields = ReservationLevelFields(newSegmentAndGroupContract._1,<br>//              newSegmentAndGroupContract._2,<br>//              arrivalDate, departureDate, newRoomCount, newLastUpdateTime,<br>//              ratePlans.head, inventoryTypeCode)<br>//            newSegmentAndGroupContract._1.copy(stayDetails = Some(getPayStayDetails(newReservationLevelFields, newReservation.reservationPayload.HotelReservation.get.hotels.head)))<br>//          })<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res not present | new non comp + paystay details - &quot; + newPayDetails)<br>        //println(&quot;5 Aug - Collation | getModifiedReservationRecord | old res not present | new comp + stay details - &quot; + compRoomSegWithStayDetails)<br><br>        newReservation<br>          .copy(reservationPayload = newReservation.reservationPayload<br>            .copy(HotelReservation = Some(newReservation<br>              .reservationPayload.HotelReservation.get<br>              .copy(totalGrossRevenue = Some(derivedFields.totalGrossRevenue),<br>                totalNetRevenue = Some(derivedFields.totalNetRevenue),<br>                segments =  Seq(segment.copy(offer = Some(segment.offer.get<br>                  .copy(productUses = newPayStayNonComp ++ newCompStay))))))))<br>    }<br>  }<br><br>//  def bifurcateNonCompNCompRoomSegments(reservation: ReservationCollationEvent)<br>//  : (Seq[(SegmentWOS3, Option[Seq[Option[GroupContractWOS3]]])], Seq[(SegmentWOS3, Option[Seq[Option[GroupContractWOS3]]])]) = {<br>//<br>//    log.info(&quot;Bifurcating NonComp and Comp Segments - bifurcateNonCompNCompRoomSegments()&quot;,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//<br>//    val roomCount = reservation.reservationPayload.HotelReservation.get<br>//      .segments.head.offer.get.productUses.head.quantity<br>//<br>//    val reservationSegments = reservation.reservationPayload.HotelReservation.get<br>//      .segments<br>//    val reservationGroupContract = reservation.reservationPayload.HotelReservation.get<br>//      .groupContracts<br>//<br>//    log.info(&quot; - Function - bifurcateNonCompNCompRoomSegments() | Value of roomCount =&gt; &quot; + roomCount,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    //added if else to handle group contracts<br>///*<br>//    val nonCompRoomSegments : Seq[(SegmentWOS3, GroupContractWOS3)]=<br>//      if(reservationGroupContract.isDefined)  reservationSegments.zip(reservationGroupContract)<br>//      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation, roomCount) &gt; 0)<br>//      else<br>//      reservationSegments.zip(reservationGroupContract)<br>//      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation, roomCount) &gt; 0)<br>//*/<br>//<br>//// verify the logic:: SC  to make group contract optional<br>//    val nonCompRoomSegments: Seq[(SegmentWOS3, Option[Seq[Option[GroupContractWOS3]]])] = {<br>//  if (reservationGroupContract.isDefined) {<br>//    reservationSegments.map(segment =&gt; (segment, reservationGroupContract))<br>//      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0)<br>//  } else {<br>//    reservationSegments.map(segment =&gt; (segment, None))<br>//      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0)<br>//<br>//  }<br>//}<br>//    //.filterNot(oldSegment =&gt; getCompRoomCount(oldSegment._1.offer.get.propertyMarketSegment, roomCount) &gt; 0)<br>//    //          .filterNot(oldSegment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.marketType.marketCode, roomCount) &gt; 0)<br>//    // verify the logic<br>//    val compRoomSegments: Seq[(SegmentWOS3, Option[Seq[Option[GroupContractWOS3]]])] = {<br>//      if (reservationGroupContract.isDefined) {<br>//        reservationSegments.map(segment =&gt; (segment, reservationGroupContract))<br>//          .filter(segment =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0)<br>//      } else {<br>//        reservationSegments<br>//          .map(segment =&gt; (segment, None))<br>//          .filter { case (segment, _) =&gt; getCompRoomCount(reservation.reservationPayload.HotelReservation.get.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0 }<br>//      }<br>//    }<br>//<br>//    log.info(&quot; - Function - bifurcateNonCompNCompRoomSegments() | Value of nonCompRoomSegments =&gt; &quot;<br>//      + nonCompRoomSegments.toString(), CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    log.info(&quot; - Function - bifurcateNonCompNCompRoomSegments() | Value of compRoomSegments =&gt; &quot;<br>//      + compRoomSegments.toString(), CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    (nonCompRoomSegments, compRoomSegments)<br>//  }<br><br>  /**<br>   * Function to get bad reservation event required in failure scenarios<br>   *<br>   * @param reservationEvent : reservation record<br>   * @param exception        : exception<br>   * @return : erroneous reservation record<br>   */<br>  def getBadReservationEntity(reservationEvent: ReservationCollationEvent,<br>                              exception: Throwable): ReservationCollationEvent = {<br><br>    log.info(&quot;Fetching these fields totalGrossRevenue and totalNetRevenue - getBadReservationEntity()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val errorEventInfo = reservationEvent<br>      .reservationInfo.copy(<br>      timeStamp = System.currentTimeMillis().toString,<br>      errorType = Seq(IrrecoverableError),<br>      message = Seq(exception.getMessage),<br>      warning = Seq(ExceptionUtils.getMessage(exception)))<br><br>    reservationEvent.copy(reservationInfo = errorEventInfo)<br>  }<br><br>  /**<br>   * Function to get committed reservation records modified with pay and stay details<br>   *<br>   * @param reservation : new reservation record for which pay and stay details are required<br>   * @return : reservation record with pay and stay details<br>   */<br>  def getCommittedReservationSegments(reservation: ReservationCollationEvent,<br>                                     ): HotelReservationWOS3 = {<br><br>    log.info(&quot;Fetching committed reservation records modified with pay and stay details - getCommittedReservationSegments()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val segment = reservation.reservationPayload.HotelReservation.get.segments.head<br><br>    val arrivalDate = reservation.reservationPayload.HotelReservation.get.timespan.get.start<br><br>    val departureDate = reservation.reservationPayload.HotelReservation.get.timespan.get.end<br><br>    val ratePlans = reservation.reservationPayload.HotelReservation.get.ratePlans<br>    val groupContracts = reservation.reservationPayload.HotelReservation.get.groupContracts<br>//    val inventoryTypeCode = reservation.reservationPayload.HotelReservation.get.products.head.inventoryTypeCode<br><br>    val requiredFields = getRequiredFieldValues(reservation)<br>//    val newRoomCount = reservation.reservationPayload.HotelReservation.get<br>//      .segments.head.offer.get.productUses.head.quantity<br><br>    val newLastUpdateTime = reservation.reservationPayload.HotelReservation.get<br>      .dateTimeStamp.get.lastModify.get<br><br>    log.info(&quot; - Function - getCommittedReservationSegments() | Value of arrivalDate =&gt; &quot; + arrivalDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getCommittedReservationSegments() | Value of departureDate =&gt; &quot; + departureDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getCommittedReservationSegments() | Value of ratePlans =&gt; &quot; + ratePlans,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    log.info(&quot; - Function - getCommittedReservationSegments() | Value of productUses =&gt; &quot; + productUses,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    log.info(&quot; - Function - getCommittedReservationSegments() | Value of newRoomCount =&gt; &quot; + newRoomCount,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getCommittedReservationSegments() | Value of newLastUpdateTime =&gt; &quot; + newLastUpdateTime,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val (nonCompRoomSegments, compRoomsSegments) = bifurcateNonCompNCompProductUses(reservation)<br>    //println(&quot;5 Aug - Collation | getCommittedReservationSegments | new res non comp - &quot; + nonCompRoomSegments)<br>    //println(&quot;5 Aug - Collation | getCommittedReservationSegments | new res comp - &quot; + compRoomsSegments)<br>//    val nonCompRoomsPayDetailsSeq = nonCompRoomSegments<br>//      .map(segmentAndGroupContract =&gt; {<br>//        val newReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//          segmentAndGroupContract._2,<br>//          arrivalDate, departureDate, newLastUpdateTime,<br>//          ratePlans)<br>//        getPayStayDetails(newReservationLevelFields, reservation.reservationPayload.HotelReservation.get.hotels.head)<br>//      })<br>    val nonCompRoomsPayStayDetailsSeq = nonCompRoomSegments.map(nonCompProductUse =&gt; {<br>      val nonCompDailyRates = nonCompProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>      val newReservationLevelFields = ReservationLevelFields(segment,<br>        groupContracts,<br>                arrivalDate, departureDate, newLastUpdateTime,<br>                ratePlans, nonCompProductUse, dailyRate)<br>      val payStayDetails = getPayStayDetails(newReservationLevelFields, reservation.reservationPayload.HotelReservation.get.hotels.head)<br>        dailyRate.copy(payDetails = Some(payStayDetails),<br>        stayDetails = Some(payStayDetails))<br>      })<br>      nonCompProductUse.copy(productRates = nonCompProductUse.productRates.copy(dailyRates = nonCompDailyRates))<br>    })<br>    val compRoomStayDetailsSeq = compRoomsSegments.map(compProductUse =&gt; {<br>      val compDailyRates = compProductUse.productRates.dailyRates.map(dailyRate =&gt; {<br>      val newReservationLevelFields = ReservationLevelFields(segment,<br>        groupContracts,<br>        arrivalDate, departureDate, newLastUpdateTime,<br>        ratePlans, compProductUse, dailyRate)<br>      val stayDetails = getPayStayDetails(newReservationLevelFields, reservation.reservationPayload.HotelReservation.get.hotels.head)<br>        dailyRate.copy(stayDetails = Some(stayDetails))<br>      })<br>      compProductUse.copy(productRates = compProductUse.productRates.copy(dailyRates = compDailyRates))<br>    })<br>//    val nonCompRoomsPayStayDetails = getPayStayDetails(<br>//      ReservationLevelFields(nonCompRoomSegments, groupContracts, arrivalDate, departureDate, newLastUpdateTime, ratePlans),<br>//      reservation.reservationPayload.HotelReservation.get.hotels.head)<br>//    val nonCompSegments = nonCompRoomSegments.copy(offer =<br>//      Some(nonCompRoomSegments.offer.get.copy(totalGrossRevenue = Some(requiredFields.totalGrossRevenue),<br>//        totalNetRevenue = Some(requiredFields.totalNetRevenue),<br>//        productUses = nonCompRoomSegments.offer.get.productUses.map(productUse =&gt;<br>//        productUse.copy(payDetails = Some(nonCompRoomsPayStayDetails),<br>//          stayDetails = Some(nonCompRoomsPayStayDetails))))))<br>//    val compRoomsStayDetails = getPayStayDetails(<br>//      ReservationLevelFields(compRoomsSegments, groupContracts, arrivalDate, departureDate, newLastUpdateTime, ratePlans),<br>//      reservation.reservationPayload.HotelReservation.get.hotels.head)<br>//    val compSegments = compRoomsSegments.copy(offer =<br>//      Some(compRoomsSegments.offer.get.copy(totalGrossRevenue = Some(requiredFields.totalGrossRevenue),<br>//        totalNetRevenue = Some(requiredFields.totalNetRevenue),<br>//        productUses = compRoomsSegments.offer.get.productUses.map(productUse =&gt;<br>//        productUse.copy(stayDetails = Some(compRoomsStayDetails))))))<br>//    val compRoomSegmentsWithStayDetails = compRoomsSegments<br>//      .map(segmentAndGroupContract =&gt; {<br>//        val newReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//          segmentAndGroupContract._2,<br>//          arrivalDate, departureDate, newLastUpdateTime,<br>//          ratePlans)<br>//        segmentAndGroupContract._1.copy(stayDetails = Some(getPayStayDetails(newReservationLevelFields, reservation.reservationPayload.HotelReservation.get.hotels.head)))<br>//      })<br>//    val nonCompRoomSegmentsWithPayStayDetails = nonCompRoomSegments<br>//      .map(segmentAndGroupContract =&gt; {<br>//        val newReservationLevelFields = ReservationLevelFields(segmentAndGroupContract._1,<br>//          segmentAndGroupContract._2,<br>//          arrivalDate, departureDate, newLastUpdateTime,<br>//          ratePlans)<br>//        val payStayDetails = getPayStayDetails(newReservationLevelFields, reservation.reservationPayload.HotelReservation.get.hotels.head<br>//        )<br>//        segmentAndGroupContract._1<br>//          .copy(payDetails = Some(nonCompRoomsPayDetailsSeq))<br>//          .copy(stayDetails = Some(payStayDetails))<br>//      })<br>    //println(&quot;5 Aug - Collation | getCommittedReservationSegments | new res pay details - &quot; + nonCompRoomsPayDetailsSeq)<br>    //println(&quot;5 Aug - Collation | getCommittedReservationSegments | new non comp + paystay details - &quot; + nonCompRoomSegmentsWithPayStayDetails)<br>    //println(&quot;5 Aug - Collation | getCommittedReservationSegments | new comp + stay details - &quot; + compRoomSegmentsWithStayDetails)<br><br>    reservation<br>      .reservationPayload.HotelReservation.get<br>      .copy(totalGrossRevenue = Some(requiredFields.totalGrossRevenue),<br>        totalNetRevenue = Some(requiredFields.totalNetRevenue),<br>        segments = Seq(segment.copy(offer = Some(segment.offer.get<br>          .copy(productUses = nonCompRoomsPayStayDetailsSeq ++ compRoomStayDetailsSeq)))))<br><br>  }<br><br>  /**<br>   * Function for get set of confirmationNumber, parentConfirmationNumber,<br>   * totalGrossRevenue and totalNetRevenue fields<br>   *<br>   * @param reservation : reservation record<br>   * @return : set of confirmationNumber, parentConfirmationNumber,<br>   *         totalGrossRevenue and totalNetRevenue fields<br>   */<br>  def getRequiredFieldValues(reservation: ReservationCollationEvent): DerivedFields = {<br><br>    log.info(&quot;Fetching these fields totalGrossRevenue and totalNetRevenue - getRequiredFieldValues()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val (totalGrossRevenue, totalNetRevenue) = getTotalGrossAndNetRevenue(reservation)<br><br>    log.info(&quot; - Function - getRequiredFieldValues() | Values fetched totalGrossRevenue &amp; totalNetRevenue - &quot;<br>      + totalGrossRevenue.toString + &quot;, &quot; + totalNetRevenue.toString,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val result = DerivedFields(totalGrossRevenue, totalNetRevenue)<br>    //println(&quot;5 Aug - Collation | getRequiredFieldValues - &quot; + result)<br>    result<br>  }<br><br>  /**<br>   * Function to get total gross and net revenues<br>   *<br>   * @param reservation : reservation record<br>   * @return : tuple of totalGrossRevenue and totalNetRevenue<br>   */<br>  def getTotalGrossAndNetRevenue(reservation: ReservationCollationEvent): (Double, Double) = {<br><br>    log.info(&quot;Calculation these fields totalGrossRevenue and totalNetRevenue - getTotalGrossAndNetRevenue()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val seqOfGrossNNetRevenue = reservation.reservationPayload.HotelReservation.get<br>      .segments.head.offer.get.productUses<br>      .flatMap(productUse =&gt; {<br><br>        productUse.productRates.dailyRates.map { rate =&gt;<br>          val (amtAfTx: Option[CurrencyAmountWOS3], amtBfTx: Option[CurrencyAmountWOS3], start: LocalDate, end: LocalDate) = if (rate.dailyRate.dailyBaseOccRate.isDefined)<br>            (rate.dailyRate.dailyBaseOccRate.get.rateDetails.rateAmount.amtAfTx, rate.dailyRate.dailyBaseOccRate.get.rateDetails.rateAmount.amtBfTx,<br>              new java.sql.Date(format.parse(rate.timespan.get.start).getTime).toLocalDate,<br>              new java.sql.Date(format.parse(rate.timespan.get.end).getTime).toLocalDate)<br>          else (rate.dailyRate.dailyTotalRate.rateDetails.rateAmount.amtAfTx, rate.dailyRate.dailyTotalRate.rateDetails.rateAmount.amtBfTx,<br>            new java.sql.Date(format.parse(rate.timespan.get.start).getTime).toLocalDate,<br>            new java.sql.Date(format.parse(rate.timespan.get.end).getTime).toLocalDate)<br>          val totalGrossRevenue = getRevenue(<br>            //modified logic<br>            (if (amtAfTx.isEmpty || amtAfTx.get.value.toInt == 0)<br>              BigDecimal(0)<br>            else amtAfTx.get.value) / math.pow(10, amtAfTx.get.numberOfDecimals.get.toDouble),<br>            productUse.quantity,<br>            start,<br>            end)<br>          //<br>          val totalNetRevenue = getRevenue(<br>            (if (amtBfTx.isEmpty || amtBfTx.get.value.toInt == 0)<br>              BigDecimal(0)<br>            else amtBfTx.get.value) / math.pow(10, amtBfTx.get.numberOfDecimals.get.toDouble),<br>            productUse.quantity, start, end<br>          )<br>          (totalGrossRevenue, totalNetRevenue)<br>        }<br>      })<br><br>    log.info(&quot; - Function - getTotalGrossAndNetRevenue() | Value of totalGrossRevenue(Sum) =&gt; &quot;<br>      + seqOfGrossNNetRevenue.map(x =&gt; x._1).sum.setScale(10, BigDecimal.RoundingMode.HALF_UP).toDouble,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getTotalGrossAndNetRevenue() | Value of totalNetRevenue(Sum) =&gt; &quot;<br>      + seqOfGrossNNetRevenue.map(x =&gt; x._2).sum.setScale(10, BigDecimal.RoundingMode.HALF_UP).toDouble,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    (seqOfGrossNNetRevenue.map(x =&gt; x._1).sum.setScale(10, BigDecimal.RoundingMode.HALF_UP).toDouble,<br>      seqOfGrossNNetRevenue.map(x =&gt; x._2).sum.setScale(10, BigDecimal.RoundingMode.HALF_UP).toDouble)<br>  }<br><br>  /**<br>   * Function to calculate the total revenue of the reservation<br>   *<br>   * @param amount        : amount<br>   * @param numberOfUnits : quantity(number of rooms)<br>   * @param segStartDate  : start or arrival date of the reservation<br>   * @param segEndDate    : end or departure date<br>   * @return<br>   */<br>  def getRevenue(amount: BigDecimal,<br>                 numberOfUnits: Int,<br>                 segStartDate: LocalDate,<br>                 segEndDate: LocalDate): BigDecimal = {<br><br>    log.info(&quot;Calculation Revenue Value - getRevenue()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getRevenue() | Value of amount =&gt; &quot; + amount,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getRevenue() | Value of numberOfUnits =&gt; &quot; + numberOfUnits,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getRevenue() | Value of segStartDate =&gt; &quot; + segStartDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getRevenue() | Value of segEndDate =&gt; &quot; + segEndDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    log.info(&quot; - Function - getRevenue() | Value of Revenue(Calculated) =&gt; &quot; + amount * numberOfUnits * ChronoUnit.DAYS.between(segStartDate, segEndDate),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    amount * numberOfUnits * ChronoUnit.DAYS.between(segStartDate, segEndDate)<br>  }<br><br>  /**<br>   * Function to get tuple of reservation key, old reservation record and new reservation record<br>   * modified with pay and stay details<br>   *<br>   * @param reservationKey : reservation key<br>   * @param newReservation : reservation record received at kafka<br>   * @param oldReservation : existing reservation record from couchbase for the reservation key<br>   * @return : combination of resevatonKey, newReservation and oldReservation updated with pay and stay details<br>   */<br>  def getReservationEventRecords(reservationKey: Option[String],<br>                                 newReservation: ReservationCollationEvent,<br>                                 oldReservation: Option[ReservationCollationEvent]): NewNOldReservationEventEntity = {<br>    Try {<br>      log.info(&quot;Fetching the tuple of reservationKey, old reservation record and new reservation record - getReservationEventRecords()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      log.info(&quot; - Function - getReservationEventRecords() | Value of reservationStatusEnum =&gt; &quot;<br>        + newReservation.reservationPayload.HotelReservation.get.reservationStatusEnum,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      newReservation.reservationPayload.HotelReservation.get<br>        .reservationStatusEnum match {<br><br>        case ReservationStatusEnum.Committed =&gt;<br><br>          //println(&quot;5 Aug - getReservationEventRecords | new committed res | starting fetching committed res segments&quot;)<br><br>          val modifiedNewReservationSegments = getCommittedReservationSegments(newReservation)<br><br>          NewNOldReservationEventEntity(reservationKey,<br>            Some(newReservation.copy(reservationPayload = newReservation<br>              .reservationPayload<br>              .copy(HotelReservation = Some(modifiedNewReservationSegments)))),<br>            None)<br><br>        case ReservationStatusEnum.Modified | ReservationStatusEnum.Cancelled =&gt;<br><br>          //println(&quot;5 Aug - getReservationEventRecords | new modified or cancelled res | starting fetching pay n stay details&quot;)<br>          val newReservationWithPayNStayDetails = getReservationWithPayNStayDetails(newReservation, oldReservation)<br><br>          NewNOldReservationEventEntity(<br>            reservationKey,<br>            Some(newReservationWithPayNStayDetails._1),<br>            newReservationWithPayNStayDetails._2)<br>      }<br>    } match {<br>      case Success(newNOldRecords) =&gt; newNOldRecords<br>      case Failure(exception) =&gt;<br><br>        log.error(&quot;Caught an Irrecoverable Error :: Exception :: getReservationEventRecords():&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        new CustomException(exception,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)).handle()<br>        NewNOldReservationEventEntity(<br>          reservationKey,<br>          Some(getBadReservationEntity(newReservation, exception)),<br>          oldReservation)<br>    }<br>  }<br><br>  /**<br>   * Spark UDF to get reservation record payLoad output of reservation event record<br>   */<br>  val getEventInfo: UserDefinedFunction = udf((payLoad: String) =&gt; {<br>    ReservationEvent(payLoad)<br>  })<br><br>  def parseOldReservationRecord(jsonString: String): Option[HotelReservationWOS3] = {<br><br>    log.info(&quot;Parsing old reservation record - parseOldReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val objectMapper = new ObjectMapper() with ScalaObjectMapper<br>    Try {<br>      JsonUtil.fromJson[HotelReservationWOS3]((objectMapper.readTree(jsonString).get(&quot;HotelReservation&quot;).toString))<br>    } match {<br>      case Success(reservationStr) =&gt;<br>        Some(reservationStr)<br>      case Failure(exception) =&gt;<br>        log.error(&quot;Failed while parsing old reservation record from couchbase :: Exception :&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        new CustomException(exception,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)).handle()<br>        None<br>    }<br>  }<br><br>  /**<br>   * Fetch the Record from the Document DB<br>   *<br>   * @param reservationKey - reservation key which is ID for the collection<br>   * @param conf           - SparkConf<br>   * @return - ReservationCollationEvent<br>   */<br>  def getRecordFromDocDB(reservationKey: String, ratePlanInd: Boolean = false)<br>                        (implicit conf: SparkConf): (Option[ReservationCollationEvent], Boolean) = {<br>    log.info(&quot;Fetching the old reservation record from the documentDB - getRecordFromDocDB()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val mongodbClient = getMongoClient(conf, DocumentDBOutput)<br><br>    log.info(&quot; - retrieving the reservation record from documentDB for the reservation key: - &quot; +<br>      reservationKey, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    Try {<br>      val databaseName = conf.get(DocumentDbOutputDatabase)<br>      val collectionName = conf.get(DocumentDbOutputCollection)<br><br>      log.info(&quot; - Reservation Key from documentDB =&gt; &quot; + reservationKey,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      val db = mongodbClient.getDatabase(databaseName)<br>      val collection = db.getCollection(collectionName)<br><br>      val updatedKey = reservationKey.split(&quot;_&quot;).toList match {<br>          case cfNum :: date :: Nil if cfNum.length &gt;= 8 =&gt; s&quot;${cfNum.take(8)}_$date&quot;<br>          case _ =&gt; reservationKey<br>        }<br><br>      val filter = or(<br>        Filters.eq(&quot;_id&quot;, reservationKey),<br>        Filters.eq(&quot;_id&quot;, updatedKey)<br>      )<br><br>      collection.find(filter).first()<br>    } match {<br>      case Success(oldReservationStr) =&gt;<br>        if (oldReservationStr != null) {<br>          val oldReservation = parseOldReservationRecord(oldReservationStr.toJson)<br>          oldReservation match {<br>            case None =&gt; (None, false)<br><br>            case Some(oldReservation) =&gt;<br>              (Some(ReservationCollationEvent(<br>                DefaultEventInfo(<br>                  EventVersion,<br>                  EventSourceValue,<br>                  EventIdentifierValue,<br>                  RecordNumberValue,<br>                  LocalDateTime.now().toString,<br>                  Map[String, String](),<br>                  EmptyString,<br>                  Seq[String](),<br>                  RetryCounterValue,<br>                  DestinationTopicValue,<br>                  Seq[String](),<br>                  Seq[String]()),<br>                ReservationCollationPayload(oldReservation.toString, &quot;json&quot;, Some(oldReservation)))), false)<br>          }<br>        } else {<br>          (None, false)<br>        }<br>      case Failure(exception: NullPointerException) =&gt; {<br>        log.error(&quot;Failed while retrieving reservation record from documentDB :: Exception :: getRecordFromDocDB(): &quot;<br>          + exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>        new CustomException(exception,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)).handle()<br>        (None, true)<br>      }<br>    }<br>  }<br><br>  def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb: &quot; + ex.printStackTrace())<br>        List.empty[ReservationCollationEvent]<br>    }<br>  }<br>  /**<br>   * Function to get latest reservation record out of list of reservation records with same reservation key.<br>   *<br>   * @param reservationKey     : reservation key<br>   * @param reservationRecords : group of reservation records for the same key<br>   * @return : latest reservation record with pay and stay details<br>   */<br>  def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                ,connectionConfigDetails: ConnectionConfigDetails): NewNOldReservationRecordsErrorEntity = {<br><br>    log.info(&quot;Fetching the latest reservation record out of list of reservation records with same reservation key - getLatestReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val reservationRecordsList = reservationRecords.toList<br>    // need to add recoverable logic here in fetchPMSfromRateplan()<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br><br>    //TODO - Add Recoverable DLQ Logic here<br>    val (docDBReservationRecord, recoverableErrFlag) = getRecordFromDocDB(reservationKey.get)<br><br>    val updatedDocDBReservation = docDBReservationRecord.map { rec =&gt;<br>      val updatedReservation = rec.reservationPayload.HotelReservation.map { res =&gt;<br>        val updatedConfirmationNumber = res.confirmationNumber.copy(value = reservationKey.get.split(&quot;_&quot;).head)<br>        res.copy(confirmationNumber = updatedConfirmationNumber)<br>      }<br>      val updatedPayload = rec.reservationPayload.copy(HotelReservation = updatedReservation)<br>      rec.copy(reservationPayload = updatedPayload)<br>    }<br><br>    val uniqueReservationRecordsList = filterExistingReservationsInDocDB(<br>      resRecWithCMS,<br>      updatedDocDBReservation)<br><br>    log.info(&quot;Sorting the Reservation Record list based n Reservation Status - getLatestReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val sortedReservationRecordList = uniqueReservationRecordsList<br>      .sortBy(record =&gt; {<br>        (record.reservationPayload.HotelReservation<br>          .get.reservationStatusEnum match {<br>          case ReservationStatusEnum.Committed =&gt; 1<br>          case ReservationStatusEnum.Modified =&gt; 2<br>          case ReservationStatusEnum.Cancelled =&gt; 3<br>        },<br>          record.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify)<br>      })<br><br>    //println(&quot;5 Aug - getLatestReservationRecord | sorted list - &quot; + sortedReservationRecordList)<br><br>    log.info(&quot;Based on Last Modify Timestamp processing the record - getLatestReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    if (sortedReservationRecordList.nonEmpty) {<br>      val newReservation = if (sortedReservationRecordList.tail.nonEmpty) {<br>        sortedReservationRecordList.tail.foldLeft(sortedReservationRecordList.head) {<br>          (res1, res2) =&gt; {<br><br><br>            log.info(&quot; - Function - getLatestReservationRecord() | Value of Res1.lastModify =&gt; &quot; +<br><br>              res1.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify,<br>              CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>            log.info(&quot; - Function - getLatestReservationRecord() | Value of Res2.lastModify =&gt; &quot; +<br>              res2.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify,<br>              CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>            if (res1.reservationPayload.HotelReservation.get<br>              .dateTimeStamp.get.lastModify.equals(res2.reservationPayload<br>              .HotelReservation.get.dateTimeStamp.get.lastModify)) {<br>              //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are equal&quot;)<br>              val result = if (updatedDocDBReservation.isEmpty)<br>                getReservationEventRecords(reservationKey, res2, None).newReservation.get<br>              else<br>                updatedDocDBReservation.get<br><br>              //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are equal | result record - &quot; + result)<br>              result<br>            }<br>            else {<br>              val newNOldReservationEntity = {<br>                if (res1.reservationPayload.HotelReservation.isDefined<br>                  &amp;&amp; res2.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>                  &amp;&amp; res1.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>                  val result = getReservationEventRecords(reservationKey, res1, Some(res2))<br>                  //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are not equal | res1 - defined, modified, res2 - committed | newNOldReservationEntity &quot; + result)<br>                  result<br>                } else {<br>                  val result = getReservationEventRecords(reservationKey, res2, Some(res1))<br>                  //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are not equal | else flow for these =&gt; res1 - defined, modified, res2 - committed | newNOldReservationEntity &quot; + result)<br>                  result<br>                }<br>              }<br><br>              if (newNOldReservationEntity.newReservation.get.getEventInfo.errorType.contains(IrrecoverableError)) {<br>                //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are not equal | irrecoverable error record &quot;)<br>                return NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(reservationKey,<br>                  Some(res2),<br>                  Some(res1),<br>                  recoverableErrFlag)))<br>              } else {<br>                //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are not equal | not an irrecoverable error record &quot;)<br>                newNOldReservationEntity.newReservation.get<br>              }<br>            }<br>          }<br>        }<br>      } else if (updatedDocDBReservation.isDefined) {<br>        //println(&quot;5 Aug - getLatestReservationRecord | sorted list tail is empty &amp; decdb is defined | docdb record - &quot; + docDBReservationRecord)<br>        updatedDocDBReservation.get<br>      } else {<br>        val newNOldReservationEntity = getReservationEventRecords(reservationKey, sortedReservationRecordList.head, None)<br>        //println(&quot;5 Aug - getLatestReservationRecord | sorted list tail is empty &amp; decdb is not defined | result new old rec - &quot; + newNOldReservationEntity)<br>        newNOldReservationEntity.newReservation.get<br>      }<br>      val oldReservationInUniqueList = sortedReservationRecordList.tail.foldLeft(sortedReservationRecordList.head) {<br>        (res1, res2) =&gt; {<br>          // res1<br>          if (res1.reservationPayload.HotelReservation.isDefined) {<br>            val result = getReservationEventRecords(reservationKey, res1, None).newReservation.get<br>            //println(&quot;5 Aug - getLatestReservationRecord | oldReservationInUniqueList | res1 defined - &quot; + result)<br>            result<br>          } else {<br>            val result = getReservationEventRecords(reservationKey, res2, Some(res1)).oldReservation.get<br>            //println(&quot;5 Aug - getLatestReservationRecord | oldReservationInUniqueList | res1 not defined - &quot; + result)<br>            result<br>          }<br>        }<br>      }<br><br>      updatedDocDBReservation match {<br>        case None =&gt;<br>          if (newReservation.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>            &amp; oldReservationInUniqueList.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>            val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>              reservationKey,<br>              Some(oldReservationInUniqueList),<br>              None, recoverableErrFlag)))<br>            //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord absent | new committed &amp; old modified - &quot; + result)<br>            result<br>          } else {<br>            val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>              reservationKey,<br>              Some(newReservation),<br>              None, recoverableErrFlag)))<br>            //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord absent | else for these =&gt; new committed &amp; old modified - &quot; + result)<br>            result<br>          }<br>        case Some(docDbRecord) =&gt;<br>          if (newReservation.reservationPayload.HotelReservation.get<br>            == docDbRecord.reservationPayload.HotelReservation.get) {<br>            val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>              None, None, None, recoverableErrFlag)))<br>            //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord present | new == docdb - &quot; + result)<br>            result<br>          } else if (newReservation.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>            &amp; docDbRecord.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>            val result = NewNOldReservationRecordsErrorEntity(None)<br>            //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord present | new committed &amp; docdb modified - &quot; + result)<br>            result<br>          } else {<br>            val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>              reservationKey,<br>              Some(newReservation),<br>              updatedDocDBReservation,<br>              recoverableErrFlag)))<br>            //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord present | else for these =&gt; new == docdb, new committed &amp; docdb modified - &quot; + result)<br>            result<br>          }<br>      }<br>    }<br>    else {<br>      val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>        None, None, None, recoverableErrFlag)))<br>      //println(&quot;5 Aug - getLatestReservationRecord | sortedlist empty | result - &quot; + result)<br>      result<br>    }<br><br>  }<br><br>  /**<br>   * Function to remove the duplicate records which already present in the couch base with<br>   * same lastUpdTime<br>   *<br>   * @param reservationRecordsList : Reservation Collation events List<br>   * @param docDBReservationRecord : Couch reservation record<br>   * @return : List of unique reservation ingestion events<br>   */<br>  def filterExistingReservationsInDocDB(reservationRecordsList: List[ReservationCollationEvent],<br>                                        docDBReservationRecord: Option[ReservationCollationEvent]): List[ReservationCollationEvent] = {<br><br>    log.info(&quot;Removing the duplicate records fetched from couchbase which has same lastUpdTime - removeDuplicateReservationRecordsFromCouchbase()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    //println(&quot;5 Aug - filterExistingReservationsInDocDB | input list - &quot; + reservationRecordsList)<br><br>    docDBReservationRecord match {<br>      case Some(docDBRecord) =&gt;<br>        log.info(&quot; - Function - filterExistingReservationsInDocDB() | Value of DocDB(lastModify) =&gt; &quot; +<br>          docDBRecord.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>        val docDBRecordLastUpdTime = docDBRecord.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify<br>        val result = reservationRecordsList.filterNot(record =&gt;<br>          record.reservationPayload.HotelReservation.get<br>            .dateTimeStamp.get.lastModify<br>            .equals(docDBRecordLastUpdTime)) ++ Some(docDBRecord)<br>        //println(&quot;5 Aug - filterExistingReservationsInDocDB | filteredlist - &quot; + result)<br>        result<br><br>      case None =&gt; reservationRecordsList<br>    }<br>  }<br>}<br><br><br><br>ReservationEventHelper<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.utils.JsonUtil<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion.exceptions.{DocumentDBException, SparkException}<br>import com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationEventsCBHelper.{convertStringToLocalDate, getModifiedReservationWithRateIndicators, isRateProgramTransient}<br>import org.apache.spark.sql._<br>import org.mongodb.scala.MongoException<br><br>import java.sql.Timestamp<br>//import com.marriott.mgpapp.spark.reservation.ingestion.helper.CustomExceptionHandler.errorHandler<br>//import com.marriott.rm.platform.exception.event.BaseException.toBaseException<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.SparkConf<br><br>import java.sql.Date<br>import java.time.format.DateTimeFormatter<br>import scala.util.{Failure, Success, Try}<br><br><br>trait ReservationEventHelper extends Serializable with ReservationIngestionHelper {<br><br>  override implicit val jobIdentifier: String = ReservationIngestion<br>  override implicit val jobName: String = Reservations<br>  override val log = new CustomLogger(this.getClass.getName)<br><br><br>  def reservationEvent(rawCollationEventDS: Dataset[ReservationEventsEntity], sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails,<br>                       connectionConfigDetails: ConnectionConfigDetails)(implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails): Dataset[ReservationKafkaEvent] = {<br><br>    log.info(&quot;Fetching the Reservation Events Details - reservationEvent()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    import sparkSession.implicits._<br><br>    val reservationCollationInputDS = rawCollationEventDS<br>      .map(reservationCollationEvent =&gt; ReservationEventsGroupEntity(ConstantGroupByKey, ReservationCollationEntity(reservationCollationEvent.reservationKey, Option(OldAndNewReservation(reservationCollationEvent.oldReservation, reservationCollationEvent.newReservation)))))<br>      .as[ReservationEventsGroupEntity]<br>    //println(&quot;5 Aug - res event | res coll input ds - &quot;)<br>    //reservationCollationInputDS.show(false)<br><br><br>    val modifiedResCollationInputWithRateIndDS = reservationCollationInputDS<br>      .mapPartitions(res =&gt; getModifiedReservationWithRateIndicators(res,connectionConfigDetails,jobConfigDetails).iterator)<br>    //println(&quot;5 Aug - res event | mod res coll input with rate ind - &quot;)<br>    //modifiedResCollationInputWithRateIndDS.show(false)<br><br><br>    val resEventProcessedDS = modifiedResCollationInputWithRateIndDS<br>      .map(record =&gt; getReservationEventRecord(record))<br>    //println(&quot;5 Aug - res event | processed res event - &quot;)<br>    //resEventProcessedDS.show(false)<br><br>    val resEventSuccessDS = resEventProcessedDS<br>      .filter(record =&gt; record._1 == ResEventSuccess)<br>      .filter(record =&gt; record._2.get.reservationSegments.nonEmpty)<br>      .map(record =&gt; getParsedResEventRecord(record._2.get, jobConfigDetails))<br>    //println(&quot;5 Aug - res event | final res event output - &quot;)<br>    //resEventSuccessDS.show(false)<br><br>    val partitionNum: Int = sparkSession.sqlContext.getConf(SparkSQLShufflePartitions).toInt<br><br>    val s3BucketName = connectionConfigDetails.bucketconfig.get(Default) + &quot;/Event&quot;<br>    val checkPointDirMap: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br>    val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>    resEventSuccessDS<br>  }<br><br>  def getReservationEventRecord(record: ReservationCollationEntity): (String, Option[ReservationEventData]) = {<br><br>    log.info(&quot;Fetching the Data of Reservation Event - getReservationEventRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    Try {<br><br>      val oldReservationRecord = record.reservations.get.oldReservation<br><br>      val newReservationRecord = record.reservations.get.newReservation<br>      //println(&quot;5 Aug - old res record - &quot; + oldReservationRecord)<br>      //println(&quot;5 Aug - new res record - &quot; + newReservationRecord)<br>      val payStayEventRecords = newReservationRecord.get.reservationStatusEnum match {<br><br>        case ReservationStatusEnum.Committed =&gt;<br>          log.info(&quot;Starting the Processing of Committed Reservation Event Data - getReservationEventRecord()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>         // processCommittedReservation(newReservationRecord.get)<br>        //Release candidate changes<br>          //println(&quot;5 Aug - starting committed flow&quot;)<br>          processModifiedReservation(newReservationRecord.get, oldReservationRecord)<br><br>        case ReservationStatusEnum.Cancelled =&gt;<br>          log.info(&quot;Starting the Processing of Cancelled Reservation Event Data - getReservationEventRecord()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>          //println(&quot;5 Aug - starting cancelled flow&quot;)<br>          processCancelledReservation(newReservationRecord.get, oldReservationRecord)<br><br>        case ReservationStatusEnum.Modified =&gt;<br>          log.info(&quot;Starting the Processing of Modified Reservation Event Data - getReservationEventRecord()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>          //println(&quot;5 Aug - starting modified flow&quot;)<br>          processModifiedReservation(newReservationRecord.get, oldReservationRecord)<br><br>      }<br>      ReservationEventData(record.reservationKey.get, payStayEventRecords)<br>    } match {<br>      case Success(reservationEventData) =&gt;<br>        log.info(&quot;Successfully Processed the Reservation Event Data - getReservationEventRecord()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        (ResEventSuccess, Some(reservationEventData))<br><br>      case Failure(exception) =&gt;<br>        exception match {<br>          case exception: MongoException =&gt;<br>            log.error(&quot;getReservationEventRecord() | Couchbase Error while processing the record: &quot; + exception.printStackTrace(),<br>              CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>            new DocumentDBException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>            (ResEventFailure, None)<br><br>          case exception: Exception =&gt;<br>            log.error(&quot;getReservationEventRecord() | Error while processing the record: &quot; + exception.printStackTrace(),<br>              CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>            new SparkException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>            (ResEventFailure, None)<br>        }<br>    }<br>  }<br><br>  def getParsedResEventRecord(record: ReservationEventData, jobConfigDetails: JobConfigurationDetails): ReservationKafkaEvent = {<br><br>    log.info(&quot;Parsing the Data of Reservation Event Record - getParsedResEventRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val event = ReservationKafkaEvent(<br>      payLoad = JsonUtil.toJson(record).toString,<br>      destinationTopic = jobConfigDetails.kafkatopicmap.get(DestinationTopicName))<br>    //val jsonEvent = JsonUtil.toJson(event).toString<br>    log.debug(&quot;getParsedResEventRecord() | Parsed Reservation Event Record: &quot;/* +  jsonEvent*/ ,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    //jsonEvent<br>    event<br>  }<br><br>//  def bifurcateNonCompNCompRoomSegments(reservation: HotelReservationWOS3)<br>//  : (Seq[SegmentWOS3], Seq[SegmentWOS3]) = {<br>//<br>//    log.info(&quot;Separating Comp and Non comp Room Segments - bifurcateNonCompNCompRoomSegments()&quot;,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>//<br>//    val roomCount = reservation.segments.head.offer.get.productUses.head.quantity<br>//    //println(&quot;5 Aug - bifurcate comp noncomp | room count - &quot; + roomCount)<br>//<br>//    val reservationSegments = reservation.segments<br>//    //println(&quot;5 Aug - bifurcate comp noncomp | res seg - &quot; + reservationSegments)<br>//<br>//    val nonCompRoomSegments = reservationSegments<br>//      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0)<br>//    //      .filterNot(oldSegment =&gt; getCompRoomCount(reservation.ratePlans.head.marketType.marketCode, roomCount) &gt; 0)<br>//    log.debug(&quot;bifurcateNonCompNCompRoomSegments() | Non Comp Room Segments: &quot; + nonCompRoomSegments,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>//<br>//    val compRoomSegments = reservationSegments<br>//      .filter(segment =&gt; getCompRoomCount(reservation.ratePlans.head.details.propertyMarketSegmentation.get, roomCount) &gt; 0)<br>////      .filter(segment =&gt; getCompRoomCount(reservation.ratePlans.head.marketType.marketCode, roomCount) &gt; 0)<br>//    log.debug(&quot;bifurcateNonCompNCompRoomSegments() | Comp Room Segments: &quot; + compRoomSegments,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>//<br>//    (nonCompRoomSegments, compRoomSegments)<br>//<br>//  }<br><br>  def processCommittedReservation(reservation: HotelReservationWOS3): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Processing the Committed Reservation Data - processCommittedReservation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    log.debug(&quot;processCommittedReservation() | Committed Reservation: &quot; + reservation,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val resFields = getReservationFields(reservation)<br>    implicit val reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields) = (resFields, resFields)<br>    //println(&quot;5 Aug - committed flow | res fields - &quot; + reservationFields)<br>    implicit val multiplier: Int = PositiveOneMultiplier<br><br>    val (nonCompRoomSegments, compRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(reservation)<br>    //println(&quot;5 Aug - committed flow | noncomp seg - &quot; + nonCompRoomSegments)<br>    //println(&quot;5 Aug - committed flow | comp seg - &quot; + compRoomSegments)<br>    log.debug(&quot;processCommittedReservation() | Non Comp Room Segments: &quot; + nonCompRoomSegments +<br>      &quot; | Comp Room Segments: &quot; + compRoomSegments,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val result = generateStayRecords(compRoomSegments, CompRoom, New) ++<br>      generateStayRecords(nonCompRoomSegments, Stay, New) ++<br>      generatePayRecords(nonCompRoomSegments, Pay, New)<br>    //println(&quot;5 Aug - committed flow | new comp stay + new noncomp stay + new noncomp pay - &quot; + result)<br>    result<br><br>  }<br><br>  def processCancelledReservation(newReservation: HotelReservationWOS3,<br>                                  oldReservation: Option[HotelReservationWOS3]): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Processing the Cancelled Reservation Data - processCancelledReservation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    oldReservation match {<br>      case Some(oldReservationRecord) =&gt;<br>        log.debug(&quot;processCancelledReservation() | Processing the Cancelled Reservation Data when Old Reservation details are provided as: &quot; + oldReservationRecord,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        val oldReservationStatus = oldReservationRecord.reservationStatusEnum.toString<br><br>        if (oldReservationStatus == Cancelled &amp;&amp; newReservation.equals(oldReservationRecord)) {<br>          //println(&quot;5 Aug - cancelled flow | old present | old res status is cancelled and new = old&quot;)<br>          Seq()<br>        } else {<br>          implicit val reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields) =<br>            (getReservationFields(newReservation), getReservationFields(oldReservationRecord))<br><br>          //println(&quot;5 Aug - cancelled flow | old present | res fields - &quot; + reservationFields)<br><br>          val (newNonCompRoomSegments, newCompRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(newReservation)<br><br>          //println(&quot;5 Aug - cancelled flow | old present | new comp seg &quot; + newCompRoomSegments)<br>          //println(&quot;5 Aug - cancelled flow | old present | new noncomp seg &quot; + newNonCompRoomSegments)<br><br>          val (oldNonCompRoomSegments, oldCompRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(oldReservationRecord)<br><br>          //println(&quot;5 Aug - cancelled flow | old present | old comp seg &quot; + oldCompRoomSegments)<br>          //println(&quot;5 Aug - cancelled flow | old present | old noncomp seg &quot; + oldNonCompRoomSegments)<br><br>          val result = generateStayRecords(newCompRoomSegments.diff(oldCompRoomSegments), CancelCompRoom, New)(reservationFields, NegativeOneMultiplier) ++<br>            // generateStayRecords(oldCompRoomSegments.diff(newCompRoomSegments), CancelCompRoomOld, Old)(reservationFields, 0)<br>            getCancellationEvents(newNonCompRoomSegments, oldNonCompRoomSegments, oldReservationStatus)<br>          //println(&quot;5 Aug - cancelled flow | old present | new cancelcomp stay records + cancellation events &quot; + result)<br>          result<br>        }<br><br>      case None =&gt;<br>        log.info(&quot;Processing the Cancelled Reservation Data with no Old Reservation details - processCancelledReservation()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        val resFields = getReservationFields(newReservation)<br>        implicit val reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields) = (resFields, resFields)<br>        //println(&quot;5 Aug - cancelled flow | old absent | res fields &quot; + reservationFields)<br>        implicit val multiplier: Int = PositiveOneMultiplier<br><br>        val (nonCompRoomSegments, compRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(newReservation)<br><br>        //println(&quot;5 Aug - cancelled flow | old absent | comp seg &quot; + compRoomSegments)<br>        //println(&quot;5 Aug - cancelled flow | old absent | noncomp seg &quot; + nonCompRoomSegments)<br><br>        val payDetailsVal = if (nonCompRoomSegments.isEmpty) Seq()<br>        else nonCompRoomSegments.flatMap(productUse =&gt; {<br>          productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>            dailyRate.payDetails.get<br>          })<br>        })<br><br>        //println(&quot;5 Aug - cancelled flow | old absent | paydetails &quot; + payDetailsVal)<br><br>        var noShowPayCheck: AnyVal = 0<br>        if (payDetailsVal.isEmpty)<br>          {<br>            noShowPayCheck = 0<br>          }<br><br>        payDetailsVal.flatMap(payDetails =&gt; {<br>          noShowPayCheck = if (payDetails.segmentStatusCode.isDefined) {<br>            if (payDetails.segmentStatusCode.get.equals(NoShow))<br>              (payDetails.roomCount * 1)<br>          }<br><br>          //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck &quot; + noShowPayCheck)<br><br>          /*This changes are done as part of one CR, where orphan cancellation with NoShow is coming,<br>           we need to treat is as committed record, and add pay and stay values<br>          so put the if condition and these 2 below methods :<br>                  generatePayRecords(nonCompRoomSegments, Pay, New) ++    generateStayRecords(nonCompRoomSegments, Stay, New)*/<br>          if (noShowPayCheck != 0) {<br>            val newPay = generatePayRecords(nonCompRoomSegments, Pay, New).map(res =&gt; {<br>              if (res.noShowPayRooms != Some(0))<br>                res<br>              else {<br>                res.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>              }<br>            })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | new pay rec &quot; + newPay)<br>            val newStay = generateStayRecords(nonCompRoomSegments, Stay, New).map(result =&gt; {<br>              if (result.noShowRooms != Some(0))<br>                result.copy(changeInLOS = Some(0))<br>              else {<br>                result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>              }<br>            })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | new stay rec &quot; + newStay)<br>            val compStay = generateStayRecords(compRoomSegments, CancelCompRoom, New).map(res =&gt; {<br>              res.copy(compRoomCount = 0)<br>            })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | comp stay rec &quot; + compStay)<br>            val nonCompCancelStay = generateStayRecords(nonCompRoomSegments, CancelStay, New).map(result =&gt; {<br>              if (result.noShowRooms != Some(0)) {<br>                result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>              }<br>              else<br>                result<br>            })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | noncomp cancelstay rec &quot; + nonCompCancelStay)<br>            val nonCompPay = generatePayRecords(nonCompRoomSegments, CancelPay, New).map(result =&gt; {<br>              if (result.noShowRooms != Some(0))<br>                result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>              else<br>                result<br>            })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | noncomp pay rec &quot; + nonCompPay)<br>            val payrecord = newPay ++ nonCompPay<br>            val stayrecord = newStay ++ nonCompCancelStay<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | pay rec &quot; + payrecord)<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | stay rec &quot; + stayrecord)<br><br>            val stayGrossRev: Double = stayrecord.map(row =&gt; (row.stayGrossRevenue)).sum<br>            val stayNetRev: Double = stayrecord.map(row =&gt; (row.stayNetRevenue)).sum<br>            val payGrossRev: Double = payrecord.map(row =&gt; row.payGrossRevenue).sum<br>            val payNetRev: Double = payrecord.map(row =&gt; (row.payNetRevenue)).sum<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | stayGrossRev &quot; + stayGrossRev)<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | stayNetRev &quot; + stayNetRev)<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | payGrossRev &quot; + payGrossRev)<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | payNetRev &quot; + payNetRev)<br>            // val numOfSegments = if(payRecords.map(row =&gt; row.inventoryType) != (stayRecords.map(row =&gt; row.inventoryType))) 2 else 4<br>            val numOfSegments = 2<br>            val newStayRecords = stayrecord.map(row =&gt; {<br>              row.copy(totalStayGrossRevenue = stayGrossRev / numOfSegments,<br>                totalStayNetRevenue = stayNetRev / numOfSegments)<br>            })<br>            val newPayRecords = payrecord.map(row =&gt; {<br>              row.copy(totalPayGrossRevenue = payGrossRev / numOfSegments,<br>                totalPayNetRevenue = payNetRev / numOfSegments)<br>            })<br>            val result = newStayRecords ++ newPayRecords ++ compStay<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck not zero | new stay + new pay + comp stay &quot; + result)<br>            result<br>          }<br>          else {<br>            val result = generateStayRecords(nonCompRoomSegments, Stay, New).map(result =&gt; {<br>              if (result.noShowRooms != Some(0)) {<br>                result.copy(changeInLOS = Some(0))<br>              }<br>              else<br>                result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>            }) ++<br>              generateStayRecords(compRoomSegments, CancelCompRoom, New).map(res =&gt; {<br>                res.copy(compRoomCount = 0)<br>              }) ++<br>              generateStayRecords(nonCompRoomSegments, CancelStay, New).map(result =&gt; {<br>                if (result.noShowRooms != Some(0))<br>                  result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>                else<br>                  result<br>              }) ++<br>              generatePayRecords(nonCompRoomSegments, CancelPay, New).map(result =&gt; {<br>                if (result.noShowRooms != Some(0))<br>                  result.copy(stayRoomCount = 0, stayGrossRevenue = 0.0, stayNetRevenue = 0.0, totalStayGrossRevenue = 0.0, totalStayNetRevenue = 0.0, cancelledStayRoomCount = 0, cancelledStayRevenue = 0.0, payRoomCount = 0, payGrossRevenue = 0.0, payNetRevenue = 0.0, totalPayGrossRevenue = 0.0, totalPayNetRevenue = 0.0, cancelledPayRoomCount = 0, cancelledPayRevenue = 0.0, noShowRooms = Some(0), noShowRevenue = Some(0.0), noShowPayRooms = Some(0), noShowPayRevenue = Some(0.0), changeInLOS = Some(0))<br>                else<br>                  result<br>              })<br>            //println(&quot;5 Aug - cancelled flow | old absent | noshowpaycheck is zero | new stay + new cancelcomp stay + new cancelstay + new cancelpay &quot; + result)<br>            result<br>          }<br><br>        })<br><br>    }<br>  }<br><br>  def processModifiedReservation(newReservation: HotelReservationWOS3,<br>                                 oldReservation: Option[HotelReservationWOS3]): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Processing the Modified Reservation Data - processModifiedReservation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>//Release candidate changes<br>    oldReservation match {<br>      case Some(oldReservationRecord) =&gt;<br>        log.debug(&quot;processModifiedReservation() | Processing the Modified Reservation Data when Old Reservation details are provided as: &quot; + oldReservationRecord,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        implicit val reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields) =<br>          (getReservationFields(newReservation), getReservationFields(oldReservationRecord))<br><br>        //println(&quot;5 Aug - modified flow | old present | res fields &quot; + reservationFields)<br><br>        val oldResStatus =  oldReservationRecord.reservationStatusEnum.toString<br>        val newResStatus = newReservation.reservationStatusEnum.toString<br><br>        //println(&quot;5 Aug - modified flow | old present | old res &quot; + oldResStatus)<br>        //println(&quot;5 Aug - modified flow | old present | new res &quot; + newResStatus)<br><br>        if  (newResStatus == &quot;Committed&quot; &amp;&amp; oldResStatus == &quot;Modified&quot; ){<br>          val resFields = getReservationFields(newReservation)<br>          implicit val reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields) = (resFields, resFields)<br>          //implicit val multiplier: Int = Zero<br>          //println(&quot;5 Aug - modified flow | old present | new committed and old modified | res fields &quot; + reservationFields)<br><br>          val (nonCompRoomSegments, compRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(newReservation)<br>          //println(&quot;5 Aug - modified flow | old present | new committed and old modified | noncomp seg&quot; + nonCompRoomSegments)<br>          //println(&quot;5 Aug - modified flow | old present | new committed and old modified | comp seg&quot; + compRoomSegments)<br><br>          //making stay/Pay metrics to 0<br>          val result = generateStayRecords(compRoomSegments, CompRoom, New)(reservationFields, Zero)++<br>            generateStayRecords(nonCompRoomSegments, Stay, New)(reservationFields, Zero)++<br>            generatePayRecords(nonCompRoomSegments, Pay, New)(reservationFields, Zero)<br>          //println(&quot;5 Aug - modified flow | old present | new committed and old modified | new comp stay + new stay + new pay rec &quot; + result)<br>          result<br><br>        }<br>        else{<br>          val (newNonCompRoomSegments, newCompRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(newReservation)<br>          //println(&quot;5 Aug - modified flow | old present | new not committed or old not modified | new noncomp seg &quot; + newNonCompRoomSegments)<br>          //println(&quot;5 Aug - modified flow | old present | new not committed or old not modified | new comp seg &quot; + newCompRoomSegments)<br><br>          val (oldNonCompRoomSegments, oldCompRoomSegments) = bifurcateNonCompNCompProductUsesForEvent(oldReservationRecord)<br>          //println(&quot;5 Aug - modified flow | old present | new not committed or old not modified | old noncomp seg &quot; + oldNonCompRoomSegments)<br>          //println(&quot;5 Aug - modified flow | old present | new not committed or old not modified | old comp seg &quot; + oldCompRoomSegments)<br>          //val oldResStatus =  oldReservationRecord.PublishReservation2.reservationStatus.mkString<br>          val result = generateStayRecords(newCompRoomSegments.diff(oldCompRoomSegments), CompRoom, New)(reservationFields, PositiveOneMultiplier) ++<br>            generateStayRecords(oldCompRoomSegments.diff(newCompRoomSegments), CompRoom, Old)(reservationFields, NegativeOneMultiplier) ++<br>            getModifiedEvents(newNonCompRoomSegments, oldNonCompRoomSegments)<br>          //println(&quot;5 Aug - modified flow | old present | new not committed or old not modified | new comp stay + old compstay + modified events &quot; + result)<br>          result<br>        }<br><br><br>        /*val (newNonCompRoomSegments, newCompRoomSegments) = bifurcateNonCompNCompRoomSegments(newReservation)<br><br>        val (oldNonCompRoomSegments, oldCompRoomSegments) = bifurcateNonCompNCompRoomSegments(oldReservationRecord)<br><br>        generateStayRecords(newCompRoomSegments.diff(oldCompRoomSegments), CompRoom, New)(reservationFields, PositiveOneMultiplier) ++<br>          generateStayRecords(oldCompRoomSegments.diff(newCompRoomSegments), CompRoom, Old)(reservationFields, NegativeOneMultiplier) ++<br>          getModifiedEvents(newNonCompRoomSegments, oldNonCompRoomSegments)<br>*/<br>      case None =&gt;<br>        log.info(&quot;Processing the Modified Reservation Data with no Old Reservation details - processModifiedReservation()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - modified flow | old absent | calling committed process &quot;)<br>        processCommittedReservation(newReservation)<br>    }<br>  }<br><br>  def getModifiedEvents(newReservationSegments: Seq[ProductUseWOS3],<br>                        oldReservationSegments: Seq[ProductUseWOS3])<br>                       (implicit reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields)): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Fetching all the Modified Reservation Events Details - getModifiedEvents()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val newPayDetails = if (newReservationSegments.isEmpty) Seq()<br>    else newReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.payDetails.get<br>      })<br>    })<br><br>    val oldPayDetails = if (oldReservationSegments.isEmpty) Seq()<br>    else oldReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.payDetails.get<br>      })<br>    })<br><br>    val newStayDetails = newReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.stayDetails.get<br>      })<br>    })<br><br>    val oldStayDetails = oldReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.stayDetails.get<br>      })<br>    })<br><br>    log.debug(&quot;getModifiedEvents() | New Pay Details: &quot; + newPayDetails +<br>      &quot; | Old Pay Details: &quot; + oldPayDetails +<br>      &quot; | New Stay Details: &quot; + newStayDetails +<br>      &quot; | Old Stay Details: &quot; + oldStayDetails,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    //println(&quot;5 Aug - modified events | new pay det &quot; + newPayDetails)<br>    //println(&quot;5 Aug - modified events | old pay det &quot; + oldPayDetails)<br>    //println(&quot;5 Aug - modified events | new stay det &quot; + newStayDetails)<br>    //println(&quot;5 Aug - modified events | old stay det &quot; + oldStayDetails)<br><br>    val payDetailChangeFlag = oldPayDetails.diff(newPayDetails).isEmpty &amp;&amp; newPayDetails.diff(oldPayDetails).isEmpty<br>    val stayDetailChangeFlag = oldStayDetails.diff(newStayDetails).isEmpty &amp;&amp; newStayDetails.diff(oldStayDetails).isEmpty<br><br>    //println(&quot;5 Aug - modified events | old vs new pay &quot; + payDetailChangeFlag)<br>    //println(&quot;5 Aug - modified events | old vs new stay &quot; + stayDetailChangeFlag)<br><br>    val payRecords = if (!payDetailChangeFlag) {<br>      val oldpay = oldPayDetails.map(oldPayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(oldPayDtl, NegativeOneMultiplier, Pay))<br>        generatePayStayRecords(oldPayDtl, Pay, NegativeOneMultiplier)(reservationFields._2, conditionalFlags, oldPayDtl.capsAndRestriction)<br>      })<br>      val newpay = newPayDetails.map(newPayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(newPayDtl, PositiveOneMultiplier, Pay))<br>        generatePayStayRecords(newPayDtl, Pay, PositiveOneMultiplier)(reservationFields._1, conditionalFlags, newPayDtl.capsAndRestriction)<br>      })<br>      oldpay ++ newpay<br>    }<br>    else<br>      Seq()<br><br>    //println(&quot;5 Aug - modified events | pay rec &quot; + payRecords)<br>    log.debug(&quot;getModifiedEvents() | Pay Record Details: &quot; + payRecords,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val stayRecords = if (!stayDetailChangeFlag) {<br>      val oldStayRecords = oldStayDetails.map(oldStayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(oldStayDtl, NegativeOneMultiplier, Stay))<br>        generatePayStayRecords(oldStayDtl, Stay, NegativeOneMultiplier)(reservationFields._2, conditionalFlags, oldStayDtl.capsAndRestriction)<br>      })<br>      //println(&quot;5 Aug - modified events | old stay rec &quot; + oldStayRecords)<br>      val newStayRecords = newStayDetails.map(newStayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(newStayDtl, PositiveOneMultiplier, Stay))<br>        // condition to check if the segment in NS, if NS negate the changeInLOS<br>        if (conditionalFlags.get.noShowRooms != Some(0)) conditionalFlags.get.changeInLOS = Some(conditionalFlags.get.changeInLOS.get * NegativeOneMultiplier)<br><br>        if (conditionalFlags.get.noShowRooms != Some(0))<br>          generatePayStayRecords(newStayDtl, Stay, Zero)(reservationFields._1, conditionalFlags, newStayDtl.capsAndRestriction)<br>        else<br>          generatePayStayRecords(newStayDtl, Stay, PositiveOneMultiplier)(reservationFields._1, conditionalFlags, newStayDtl.capsAndRestriction)<br><br>      })<br>      //println(&quot;5 Aug - modified events | new stay rec &quot; + newStayRecords)<br>      val stayGrossRev: Double = newStayRecords.map(_.stayGrossRevenue).sum<br>      val stayNetRev: Double = newStayRecords.map(_.stayNetRevenue).sum<br><br>      val numOfSegments = 1<br>      val oldStayGrossRev: Double = oldStayRecords.map(_.stayGrossRevenue).sum<br>      val oldStayNetRev: Double = oldStayRecords.map(_.stayNetRevenue).sum<br>      val newRec = newStayRecords.map(row =&gt; {<br>        row.copy(totalStayGrossRevenue = stayGrossRev / numOfSegments,<br>          totalStayNetRevenue = stayNetRev / numOfSegments)<br>      })<br>      val oldRec = oldStayRecords.map(row =&gt; {<br>        row.copy(totalStayGrossRevenue = oldStayGrossRev / numOfSegments,<br>          totalStayNetRevenue = oldStayNetRev / numOfSegments)<br>      })<br>      //println(&quot;5 Aug - modified events | stayGrossRev &quot; + stayGrossRev)<br>      //println(&quot;5 Aug - modified events | stayNetRev &quot; + stayNetRev)<br>      //println(&quot;5 Aug - modified events | oldStayGrossRev &quot; + oldStayGrossRev)<br>      //println(&quot;5 Aug - modified events | oldStayNetRev &quot; + oldStayNetRev)<br>      //println(&quot;5 Aug - modified events | newRec &quot; + newRec)<br>      //println(&quot;5 Aug - modified events | oldRec &quot; + oldRec)<br>      newRec ++ oldRec<br>    }<br>    else<br>      Seq()<br><br>    //println(&quot;5 Aug - modified events | stay rec &quot; + stayRecords)<br>    log.debug(&quot;getModifiedEvents() | Stay Record Details: &quot; + stayRecords,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    log.debug(&quot;getModifiedEvents() | Pay and Stay Record Details: &quot; + (payRecords ++ stayRecords).distinct,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    (payRecords ++ stayRecords).distinct<br>  }<br><br>  def getCancellationEvents(newReservationSegments: Seq[ProductUseWOS3],<br>                            oldReservationSegments: Seq[ProductUseWOS3],<br>                            oldReservationStatus: String)<br>                           (implicit reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields)): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Fetching all the Cancelled Reservation Events Details - getCancellationEvents()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val newPayDetails = if (newReservationSegments.isEmpty) Seq()<br>    else newReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.payDetails.get<br>      })<br>    })<br><br>    val oldPayDetails = if (oldReservationSegments.isEmpty) Seq()<br>    else oldReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.payDetails.get<br>      })<br>    })<br><br>    val newStayDetails = newReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.stayDetails.get<br>      })<br>    })<br><br>    val oldStayDetails = oldReservationSegments.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>        dailyRate.stayDetails.get<br>      })<br>    })<br><br>    log.debug(&quot;getCancellationEvents() | New Pay Details: &quot; + newPayDetails +<br>      &quot; | Old Pay Details: &quot; + oldPayDetails +<br>      &quot; | New Stay Details: &quot; + newStayDetails +<br>      &quot; | Old Stay Details: &quot; + oldStayDetails,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    //println(&quot;5 Aug - cancellation events | new pay det &quot; + newPayDetails)<br>    //println(&quot;5 Aug - cancellation events | old pay det &quot; + oldPayDetails)<br>    //println(&quot;5 Aug - cancellation events | new stay det &quot; + newStayDetails)<br>    //println(&quot;5 Aug - cancellation events | old stay det &quot; + oldStayDetails)<br><br>    val segcode = if (newStayDetails.nonEmpty &amp;&amp; newStayDetails.head.segmentStatusCode.isDefined)<br>      newStayDetails.head.segmentStatusCode.get<br>    else if (oldStayDetails.nonEmpty &amp;&amp; oldStayDetails.head.segmentStatusCode.isDefined) {<br>      oldStayDetails.head.segmentStatusCode.get<br>    }<br>    else &quot;&quot;<br><br>    //println(&quot;5 Aug - cancellation events | segcode &quot; + segcode)<br><br>    /*val oldReservationWithNonNSUpgrade: Boolean = if ((oldReservationStatus == Modified || oldReservationStatus == Committed)<br>      &amp;&amp; (newStayDetails.nonEmpty &amp;&amp; newStayDetails.head.segmentStatusCode.isDefined &amp;&amp; newStayDetails.head.segmentStatusCode.get != NoShow)) true else false<br>  */<br><br>    val oldReservationWithNonNSUpgrade: Boolean = if ((oldReservationStatus == Modified || oldReservationStatus == Committed)<br>      &amp;&amp; (segcode != NoShow)) true else false<br><br>    //println(&quot;5 Aug - cancellation events | res - modified or committed and non ns&quot; + oldReservationWithNonNSUpgrade)<br><br>    val status = if (oldStayDetails.nonEmpty &amp;&amp; oldStayDetails.head.segmentStatusCode.isDefined) {<br>      oldStayDetails.head.segmentStatusCode.get<br>    }<br>    else &quot;&quot;<br><br>    //println(&quot;5 Aug - cancellation events | seg status &quot; + status)<br>    //indicator to check modified NS records<br>    val resWithModNS: Boolean = if ((oldReservationStatus == Modified) &amp;&amp; (status == NoShow)) true else false<br><br>    //println(&quot;5 Aug - cancellation events | res - modified and noshow&quot; + resWithModNS)<br><br><br>    if (oldReservationStatus == Cancelled) {<br>      val cancelPayRecords = if (!newPayDetails.equals(oldPayDetails))<br>        oldPayDetails.map(oldPayRecord =&gt; generatePayStayRecords(oldPayRecord, CancelPay, NegativeOneMultiplier)(reservationFields._2, None, oldPayRecord.capsAndRestriction)) ++<br>          newPayDetails.map(newPayRecord =&gt; generatePayStayRecords(newPayRecord, CancelPay, PositiveOneMultiplier)(reservationFields._2, None, newPayRecord.capsAndRestriction))<br>      else<br>        Seq()<br><br>      //println(&quot;5 Aug - cancellation events | old res cancelled | cancel pay&quot; + cancelPayRecords)<br><br>      val cancelStayRecords = if (!newStayDetails.equals(oldStayDetails))<br>        oldStayDetails.map(oldStayRecord =&gt; generatePayStayRecords(oldStayRecord, CancelStay, NegativeOneMultiplier)(reservationFields._2, None, oldStayRecord.capsAndRestriction)) ++<br>          newStayDetails.map(newStayRecord =&gt; generatePayStayRecords(newStayRecord, CancelStay, PositiveOneMultiplier)(reservationFields._2, None, newStayRecord.capsAndRestriction))<br>      else<br>        Seq()<br><br>      //println(&quot;5 Aug - cancellation events | old res cancelled | cancel stay&quot; + cancelStayRecords)<br><br>      log.debug(&quot;getCancellationEvents() | Cancellation Events Details when Old Reservation provided is Cancelled: &quot;<br>        + (cancelPayRecords ++ cancelStayRecords).distinct,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>      (cancelPayRecords ++ cancelStayRecords).distinct<br>    }<br>    else {<br>      val payOldRecords = oldPayDetails<br>        .map(oldPayDtl =&gt; {<br>          val conditionalFlags = Some(getConditionalFlags(oldPayDtl, NegativeOneMultiplier, Pay))<br>          if (conditionalFlags.get.noShowPayRooms != Some(0)) {<br>            generatePayStayRecords(oldPayDtl, CancelPay, Zero)(reservationFields._2, conditionalFlags, oldPayDtl.capsAndRestriction)<br>          }<br>          else {<br>            generatePayStayRecords(oldPayDtl, Pay, NegativeOneMultiplier)(reservationFields._2, conditionalFlags, oldPayDtl.capsAndRestriction)<br>          }<br>        })<br>      //println(&quot;5 Aug - cancellation events | old res not cancelled | old pay&quot; + payOldRecords)<br>      val payNewRecords = newPayDetails.map(newPayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(newPayDtl, PositiveOneMultiplier, Pay))<br>        generatePayStayRecords(newPayDtl, CancelPay, PositiveOneMultiplier)(reservationFields._1, conditionalFlags, newPayDtl.capsAndRestriction)<br>      })<br>      //println(&quot;5 Aug - cancellation events | old res not cancelled | new pay&quot; + payNewRecords)<br>      val payRecords = payOldRecords ++ payNewRecords<br>      log.debug(&quot;getCancellationEvents() | Pay Record Details when Old Reservation provided is not Cancelled: &quot; + payRecords,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>      val stayOldRecords = oldStayDetails<br>        .map(oldStayDtl =&gt; {<br>          val conditionalFlags = Some(getConditionalFlags(oldStayDtl, NegativeOneMultiplier, Stay))<br>          conditionalFlags.get.changeInLOS = Some(0)<br>          if (conditionalFlags.get.noShowRooms != Some(0)) {<br>            if (resWithModNS)<br>              generatePayStayRecords(oldStayDtl, CancelStay, Zero)(reservationFields._2, conditionalFlags, oldStayDtl.capsAndRestriction)<br>            else<br>              generatePayStayRecords(oldStayDtl, CancelStay, NegativeOneMultiplier)(reservationFields._2, conditionalFlags, oldStayDtl.capsAndRestriction)<br>          } else {<br>            generatePayStayRecords(oldStayDtl, Stay, NegativeOneMultiplier)(reservationFields._2, conditionalFlags, oldStayDtl.capsAndRestriction)<br><br>          }<br>        })<br>      //println(&quot;5 Aug - cancellation events | old res not cancelled | old stay&quot; + stayOldRecords)<br>      val stayNewRecords = newStayDetails.map(newStayDtl =&gt; {<br>        val conditionalFlags = Some(getConditionalFlags(newStayDtl, PositiveOneMultiplier, Stay))<br>        conditionalFlags.get.changeInLOS = Some(0)<br>        generatePayStayRecords(newStayDtl, CancelStay, PositiveOneMultiplier)(reservationFields._1, conditionalFlags, newStayDtl.capsAndRestriction)<br>      })<br>      //println(&quot;5 Aug - cancellation events | old res not cancelled | new stay&quot; + stayNewRecords)<br><br>      val stayRecords = stayOldRecords ++ stayNewRecords<br>      log.debug(&quot;getCancellationEvents() | Stay Record Details when Old Reservation provided is not Cancelled: &quot; + stayRecords,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>      if (oldReservationWithNonNSUpgrade) {<br>        val oldPayGrossRev = payOldRecords.map(_.payGrossRevenue).sum<br>        val oldPayNetRev = payOldRecords.map(_.payNetRevenue).sum<br>        val newPayGrossRev = payNewRecords.map(_.payGrossRevenue).sum<br>        val newPayNetRev = payNewRecords.map(_.payNetRevenue).sum<br>        val oldStayGrossRev = stayOldRecords.map(_.stayGrossRevenue).sum<br>        val oldStayNetRev = stayOldRecords.map(_.stayNetRevenue).sum<br>        val newStayGrossRev = stayNewRecords.map(_.stayGrossRevenue).sum<br>        val newStayNetRev = stayNewRecords.map(_.stayNetRevenue).sum<br><br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | oldPayGrossRev&quot; + oldPayGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | oldPayNetRev&quot; + oldPayNetRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | newPayGrossRev&quot; + newPayGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | newPayNetRev&quot; + newPayNetRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | oldStayGrossRev&quot; + oldStayGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | oldStayNetRev&quot; + oldStayNetRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | newStayGrossRev&quot; + newStayGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | newStayNetRev&quot; + newStayNetRev)<br><br>        val result = (payOldRecords.map(_.copy(totalPayGrossRevenue = oldPayGrossRev, totalPayNetRevenue = oldPayNetRev)) ++<br>          payNewRecords.map(_.copy(totalPayGrossRevenue = newPayGrossRev, totalPayNetRevenue = newPayNetRev)) ++<br>          stayOldRecords.map(_.copy(totalStayGrossRevenue = oldStayGrossRev, totalStayNetRevenue = oldStayNetRev)) ++<br>          stayNewRecords.map(_.copy(totalStayGrossRevenue = newStayGrossRev, totalStayNetRevenue = newStayNetRev))).distinct<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and nonshow | old pay + new pay + old stay + new stay &quot; + result)<br>        result<br><br>      } else {<br>        val stayGrossRev: Double = stayRecords.map(_.stayGrossRevenue).sum<br>        val stayNetRev: Double = stayRecords.map(_.stayNetRevenue).sum<br>        val payGrossRev: Double = payRecords.map(_.payGrossRevenue).sum<br>        val payNetRev: Double = payRecords.map(row =&gt; (row.payNetRevenue)).sum<br>        val numOfSegments = 2<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and no show | stayGrossRev&quot; + stayGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and no show | stayNetRev&quot; + stayNetRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and no show | payGrossRev&quot; + payGrossRev)<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and no show | payNetRev&quot; + payNetRev)<br><br>        val result = stayRecords.map(_.copy(totalStayGrossRevenue = stayGrossRev / numOfSegments,<br>          totalStayNetRevenue = stayNetRev / numOfSegments)) ++<br>          payRecords.map(_.copy(totalPayGrossRevenue = payGrossRev / numOfSegments,<br>            totalPayNetRevenue = payNetRev / numOfSegments))<br>        //println(&quot;5 Aug - cancellation events | old res not cancelled and no show | stay + pay &quot; + result)<br>        result<br>      }<br>    }<br>  }<br><br>  def generateStayRecords(productUses: Seq[ProductUseWOS3],<br>                          recordType: String,<br>                          reservationVersion: String)<br>                         (implicit reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields),<br>                          multiplier: Int): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Fetching all the Stay Record Details - generateStayRecords()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    productUses.flatMap(productUse =&gt; {<br>      productUse.productRates.dailyRates.map(dailyRate =&gt; {<br>      implicit val capsAndRestriction: CapsAndRestrictionWOS3 = productUse.capsAndRestriction.getOrElse(CapsAndRestrictionWOS3(Some(&quot;&quot;)))<br>      implicit val conditionalFlags: Option[ConditionalFlags] = Some(getConditionalFlags(dailyRate.stayDetails.get, multiplier, Stay))<br>      //println(&quot;5 Aug - gen stay | capsAndRestriction&quot; + capsAndRestriction)<br>      //println(&quot;5 Aug - gen stay | conditionalFlags&quot; + conditionalFlags)<br>      reservationVersion match {<br>        case New =&gt;<br>          log.info(&quot;Starting the Generation process of New Stay Record Details - generateStayRecords()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>          //println(&quot;5 Aug - gen stay | starting to gen pay stay for new res&quot;)<br>          generatePayStayRecords(dailyRate.stayDetails.get, recordType, multiplier)(reservationFields._1, conditionalFlags, capsAndRestriction)<br><br>        case Old =&gt;<br>          log.info(&quot;Starting the Generation process of Old Stay Record Details - generateStayRecords()&quot;,<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>          //println(&quot;5 Aug - gen stay | starting to gen pay stay for old res&quot;)<br>          generatePayStayRecords(dailyRate.stayDetails.get, recordType, multiplier)(reservationFields._2, None, capsAndRestriction)<br>      }<br>      })<br>    })<br>  }<br><br>  def generatePayRecords(productUses: Seq[ProductUseWOS3],<br>                         recordType: String,<br>                         reservationVersion: String)<br>                        (implicit reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields),<br>                         multiplier: Int): Seq[ResEventPayStayDetails] = {<br><br>    log.info(&quot;Fetching all the Pay Record Details - generatePayRecords()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    if (productUses.isEmpty) {<br>      Seq.empty[ResEventPayStayDetails]<br>    } else {<br>      productUses.flatMap(productUse =&gt; {<br>        productUse.productRates.dailyRates.flatMap(dailyRate =&gt; {<br>        val payDetails = dailyRate.payDetails<br>        if (payDetails.isEmpty) {<br>          Seq.empty[ResEventPayStayDetails]<br>        } else {<br>          implicit val capsAndRestriction: CapsAndRestrictionWOS3 = productUse.capsAndRestriction.getOrElse(CapsAndRestrictionWOS3(Some(&quot;&quot;)))<br>          val conditionalFlags = Some(getConditionalFlags(payDetails.get, multiplier, Pay))<br>          //println(&quot;5 Aug - gen pay | conditionalFlags&quot; + conditionalFlags)<br>          reservationVersion match {<br>            case New =&gt;<br>              log.info(&quot;Starting the Generation process of New Pay Record Details - generatePayRecords()&quot;,<br>                CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>              //println(&quot;5 Aug - gen pay | starting to gen pay stay for new res&quot;)<br>              Seq(generatePayStayRecords(payDetails.get, recordType, multiplier)(reservationFields._1, conditionalFlags, capsAndRestriction))<br><br>            case Old =&gt;<br>              log.info(&quot;Starting the Generation process of Old Pay Record Details - generatePayRecords()&quot;,<br>                CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>              //println(&quot;5 Aug - gen pay | starting to gen pay stay for old res&quot;)<br>              Seq(generatePayStayRecords(payDetails.get, recordType, multiplier)(reservationFields._2, None, capsAndRestriction))<br>          }<br>        }<br>        })<br>      })<br>    }<br>  }<br><br>  /**<br>   * Update Complimentary Room Count based on the market code.<br>   *<br>   * @return<br>   */<br>  /*def getCompRoomCount(mktCode: String, roomCount: Int): Int = {<br><br>    val initialMktCode = mktCode.substring(0, 2)<br>    logger.debug(&quot;Segment Initial Market Code::&quot; + initialMktCode)<br>    logger.debug(&quot;Segment Initial Room Count::&quot; + roomCount)<br><br>    val compRoomCount = initialMktCode match {<br>      case code if code.matches(NumericRegex) =&gt;<br>        logger.debug(&quot;Integer Prefix Code::&quot; + code)<br>        code.toInt match {<br>          // case marketCode if InitCompMktCode until FinalCompMktCode contains marketCode =&gt;<br>          case marketCode if marketCode equals 62 =&gt;<br>            logger.debug(&quot;Integer Prefix Cod &quot; + marketCode + &quot; roomCount ::&quot; + roomCount)<br>            if (roomCount &gt; 0)<br>              roomCount<br>            else<br>              0<br>          case _ =&gt; 0<br>        }<br>      case _ =&gt;<br>        0<br>    }<br>    compRoomCount<br>  }*/<br><br>  def getConditionalFlags(eventPayStayDetails: PayStayDetailsWOS3, multiplier: Int, segmentType: String)<br>                         (implicit reservationFields: (ReservationEventLevelFields, ReservationEventLevelFields)): ConditionalFlags = {<br><br>    log.info(&quot;Generating the Conditional Flags according to Pay and Stay Record Details - getConditionalFlags()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    //Same day arrival is when arrival date is same as reservation create date<br>    val sameDayCheckIn = if (reservationFields._1.reservationArrivalDate == reservationFields._1.creationDateTime)<br>      eventPayStayDetails.roomCount * multiplier<br>    else 0<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when arrival date is same as reservation create date: &quot; + sameDayCheckIn,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | sameDayCheckIn &quot; + sameDayCheckIn)<br><br>    //extending the stay<br>    //new reservation LOS is greater than old reservation<br>    //lastUpdTime of new reservation = departure of old reservation<br>    //returning total number of days extended<br>    val unexpectedStay = if (reservationFields._2.totalLOS &lt; reservationFields._1.totalLOS)<br>      if (convertStringToTimestamp(reservationFields._1.lastUpdateTime).toLocalDateTime.toLocalDate.isAfter(convertStringToDate(reservationFields._1.reservationArrivalDate)))<br>        if (convertStringToTimestamp(reservationFields._1.lastUpdateTime).toLocalDateTime.toLocalDate.equals(convertStringToDate(reservationFields._2.reservationDepartureDate)))<br>          getLOS(reservationFields._2.reservationDepartureDate, reservationFields._1.reservationDepartureDate) * multiplier<br>        else 0<br>      else 0<br>    else 0<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when there is change in stay period of the reservation: &quot; + unexpectedStay,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | unexpectedStay &quot; + unexpectedStay)<br><br><br>    val changeInLos = if (reservationFields._1.reservationStatus != Committed)<br>      if (eventPayStayDetails.propertyMarketSegmentation.substring(0, 2).matches(NumericRegex))<br>        if (!(eventPayStayDetails.propertyMarketSegmentation.substring(0, 2).toInt &gt; 60 &amp;&amp;<br>          eventPayStayDetails.propertyMarketSegmentation.substring(0, 2).toInt &lt; 69) &amp;&amp;<br>          isRateProgramTransient(eventPayStayDetails.propertyMarketSegmentation))<br>          eventPayStayDetails.roomCount * multiplier<br>        else 0<br>      else 0<br>    else 0<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when there is change in reservation details: &quot; + changeInLos,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | changeInLos &quot; + changeInLos)<br><br>    //if segmentStatusCode from input is &quot;NS&quot; then no_show<br>    val (noShow, noShowRev) = if (segmentType.equals(Stay))<br>      if (eventPayStayDetails.segmentStatusCode.isDefined)<br>        if (eventPayStayDetails.segmentStatusCode.get.toString.equals(NoShow))<br>          (eventPayStayDetails.roomCount * multiplier,<br>            (eventPayStayDetails.roomCount *<br>              eventPayStayDetails.baseAmount / math.pow(10, eventPayStayDetails.decimalPlaces) *<br>              getLOS(eventPayStayDetails.startDate, eventPayStayDetails.endDate) * multiplier).toDouble)<br>        else (0, 0.0)<br>      else (0, 0.0)<br>    else (0, 0.0)<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when reservation is made under No Show Stay Segment: &quot; + (noShow, noShowRev),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | noShow &quot; + noShow)<br>    //println(&quot;5 Aug - gen con flag | noShowRev &quot; + noShowRev)<br><br>    val (noShowPay, noShowPayRev) = if (segmentType.equals(Pay))<br>      if (eventPayStayDetails.segmentStatusCode.isDefined)<br>        if (eventPayStayDetails.segmentStatusCode.get.toString.equals(NoShow))<br>          (eventPayStayDetails.roomCount * multiplier,<br>            (eventPayStayDetails.roomCount *<br>              eventPayStayDetails.baseAmount / math.pow(10, eventPayStayDetails.decimalPlaces) *<br>              getLOS(eventPayStayDetails.startDate, eventPayStayDetails.endDate) * multiplier).toDouble)<br>        else (0, 0.0)<br>      else (0, 0.0)<br>    else (0, 0.0)<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when reservation is made under No Show Pay Segment: &quot; + (noShowPay, noShowPayRev),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | noShowPay &quot; + noShowPay)<br>    //println(&quot;5 Aug - gen con flag | noShowPayRev &quot; + noShowPayRev)<br><br>    //LOS is less<br>    //res started<br>    //lastUpdTime is equals departure date<br>    //1 - new . 2 - old<br>    val earlyCheckout = if (reservationFields._2.totalLOS &gt; reservationFields._1.totalLOS)<br>      if (convertStringToTimestamp(reservationFields._1.lastUpdateTime).toLocalDateTime.toLocalDate.isAfter(convertStringToDate(reservationFields._1.reservationArrivalDate)))<br>        if (convertStringToTimestamp(reservationFields._1.lastUpdateTime).toLocalDateTime.toLocalDate.equals(convertStringToDate(reservationFields._1.reservationDepartureDate)))<br>          getLOS(reservationFields._1.reservationDepartureDate, reservationFields._2.reservationDepartureDate) * multiplier<br>        else 0<br>      else 0<br>    else 0<br>    log.debug(&quot;getConditionalFlags() | Value of Conditional Flag when lastUpdTime of reservation is equal to departure date: &quot; + earlyCheckout,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //println(&quot;5 Aug - gen con flag | earlyCheckout &quot; + earlyCheckout)<br><br>    if (segmentType.equals(Pay)) {<br>      val result = ConditionalFlags(<br>        sameDayCheckIn = Some(sameDayCheckIn),<br>        unexpectedStay = Some(unexpectedStay),<br>        earlyCheckout = Some(earlyCheckout),<br>        noShowRooms = Some(0),<br>        noShowRevenue = Some(0),<br>        noShowPayRooms = Some(noShowPay),<br>        noShowPayRevenue = Some(noShowPayRev),<br>        changeInLOS = Some(0))<br>      //println(&quot;5 Aug - gen con flag | pay con flags &quot; + result)<br>      result<br>    }<br>    else {<br>      val result = ConditionalFlags(<br>        sameDayCheckIn = Some(0),<br>        unexpectedStay = Some(0),<br>        earlyCheckout = Some(0),<br>        noShowRooms = Some(noShow),<br>        noShowRevenue = Some(noShowRev),<br>        noShowPayRooms = Some(0),<br>        noShowPayRevenue = Some(0),<br>        changeInLOS = Some(changeInLos))<br>      //println(&quot;5 Aug - gen con flag | stay con flags &quot; + result)<br>      result<br>    }<br>  }<br><br><br>  /**<br>   * Retrieves the reservation fields for old/new reservation.<br>   */<br>  def getReservationFields(reservation: HotelReservationWOS3): ReservationEventLevelFields = {<br><br>    log.info(&quot;Retrieving the Reservation fields for the provided Reservation Event - getReservationFields()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val confirmationNumber = reservation.confirmationNumber.value<br>    log.debug(&quot;getReservationFields() | Confirmation Number of Reservation Event: &quot; + confirmationNumber,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    //    val parentConfirmationNumber = reservation.parentConfirmationNumber<br><br>    val creationDate = reservation.dateTimeStamp.get.create.get<br>    log.debug(&quot;getReservationFields() | Formatted Reservation Created date: &quot; + creationDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val propCode = reservation.hotels.head.propertyCode.code<br>    log.debug(&quot;getReservationFields() | Property Code for Reservation: &quot; + propCode,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val lastUpdateTime = reservation.dateTimeStamp.get.lastModify.get<br>    log.debug(&quot;getReservationFields() | Formatted Reservation Last Modified time: &quot; + lastUpdateTime,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>//    val roomCount = reservation.segments.head.offer.get.productUses.head.quantity<br>//    log.debug(&quot;getReservationFields() | Room Count of Reservation: &quot; + roomCount,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val totalGrossRevenue = reservation.totalGrossRevenue<br>    log.debug(&quot;getReservationFields() | Gross Revenue of Reservation: &quot; + totalGrossRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val totalNetRevenue = reservation.totalNetRevenue<br>    log.debug(&quot;getReservationFields() | Net Revenue of Reservation: &quot; + totalNetRevenue,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val reservationStatus = reservation.reservationStatusEnum.toString<br>    log.debug(&quot;getReservationFields() | Status of Reservation: &quot; + reservationStatus,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val resArrivalDate = reservation.timespan.get.start<br>    log.debug(&quot;getReservationFields() | Arrival Date for Reservation: &quot; + resArrivalDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val resDepartureDate = reservation.timespan.get.end<br>    log.debug(&quot;getReservationFields() | Departure Date for Reservation: &quot; + resDepartureDate,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val totalLOS = getLOS(resArrivalDate, resDepartureDate)<br>    log.debug(&quot;getReservationFields() | LOS for Reservation: &quot; + totalLOS,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val brandCode = reservation.hotels.head.propertyBrand<br>    log.debug(&quot;getReservationFields() | Brand Code of Reservation: &quot; + brandCode,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>//    val inventoryType = reservation.products.head.inventoryTypeCode<br>//    log.debug(&quot;getReservationFields() | Room Pool: &quot; + inventoryType,<br>//      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    ReservationEventLevelFields(<br>      confirmationNumber.toString,<br>      //      parentConfirmationNumber.toString,<br>      creationDate,<br>      resArrivalDate,<br>      resDepartureDate,<br>      reservationStatus,<br>      propCode,<br>      lastUpdateTime,<br>      totalGrossRevenue.get,<br>      totalNetRevenue.get,<br>//      inventoryType,<br>//      roomCount,<br>      totalLOS,<br>      brandCode.code.get)<br>  }<br><br>  def generatePayStayRecords(payOrStayDetails: PayStayDetailsWOS3,<br>                             recordType: String,<br>                             multiplier: Int)<br>                            (implicit reservationFields: ReservationEventLevelFields,<br>                             conditionalFlags: Option[ConditionalFlags],<br>                             capsAndRestriction: CapsAndRestrictionWOS3): ResEventPayStayDetails = {<br><br>    log.info(&quot;Fetching all Pay and Stay Record Details - generatePayStayRecords()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val conFlags = conditionalFlags match {<br>      case Some(flags) =&gt; flags<br>      case None =&gt; ConditionalFlags()<br>    }<br><br>    //println(&quot;5 Aug - gen pay stay | con flags &quot; + conFlags)<br><br>    val resEventPayStayDetails = ResEventPayStayDetails(<br>      confirmationNumber = reservationFields.confirmationNumber.toString.trim,<br>      //      parentConfirmationNumber = reservationFields.parentConfirmationNumber.toString.trim,<br>      creationDate = reservationFields.creationDateTime.toString,<br>      reservationStatus = reservationFields.reservationStatus,<br>      propertyCode = reservationFields.propertyCode,<br>      lastUpdateTime = payOrStayDetails.lastUpdateTime.trim,<br>      reservationArrivalDate = payOrStayDetails.arrivalDate.toString,<br>      groupBlockCode = if (payOrStayDetails.groupBlockCode.isDefined) payOrStayDetails.groupBlockCode.get else &quot;&quot;,<br>      ratePlan = payOrStayDetails.ratePlanCode,<br>      //added priceplancode<br>      pricePlan = payOrStayDetails.priceGridCode,<br>      inventoryTypeCode = payOrStayDetails.inventoryTypeCode,<br>      marketSegment = payOrStayDetails.propertyMarketSegmentation,<br>      currencyCode = payOrStayDetails.currencyCode,<br>      segmentArrivalDate = payOrStayDetails.startDate.toString,<br>      segmentDepartureDate = payOrStayDetails.endDate.toString,<br>      groupType = if (payOrStayDetails.groupType.isDefined) payOrStayDetails.groupType.get else &quot;&quot;,<br>      groupCfNumber = if (payOrStayDetails.groupCfNumber.isDefined) payOrStayDetails.groupCfNumber.get else &quot;&quot;,<br>      //      groupSequenceNumber = if (Option(payOrStayDetails.groupSequenceNumber).isDefined)<br>      //        payOrStayDetails.groupSequenceNumber.toString else &quot;&quot;,<br>      segmentLOS = getLOS(payOrStayDetails.startDate, payOrStayDetails.endDate).toString,<br>      segmentLOSOffset = getLOS(payOrStayDetails.arrivalDate, payOrStayDetails.startDate) + 1,<br>      totalLOS = getLOS(payOrStayDetails.arrivalDate, payOrStayDetails.departureDate),<br>//      allowCaps = capsAndRestriction.allowCaps.get,<br>//      allowRstr = capsAndRestriction.allowRstr.get,<br>//      mirrorInvRstr = capsAndRestriction.mirrorInvRstr.get,<br>      guarAvail = capsAndRestriction.guarAvail.getOrElse(&quot;&quot;))<br>//      rateType = capsAndRestriction.rateType.get)<br><br>    //println(&quot;5 Aug - gen pay stay | resEventPayStayDetails &quot; + resEventPayStayDetails)<br><br>    val compRoomCnt = getCompRoomCount(payOrStayDetails.propertyMarketSegmentation, payOrStayDetails.roomCount)<br>    //println(&quot;5 Aug - gen pay stay | comp room count &quot; + compRoomCnt)<br><br>    recordType match {<br>      case Stay =&gt;<br>        log.info(&quot;Fetching Stay Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing stay rec&quot;)<br>        resEventPayStayDetails<br>          .copy(noShowRooms = conFlags.noShowRooms,<br>            noShowRevenue = conFlags.noShowRevenue,<br>            sameDayCheckIn = conFlags.sameDayCheckIn,<br>            earlyCheckout = conFlags.earlyCheckout,<br>            unexpectedStay = conFlags.unexpectedStay,<br>            changeInLOS = conFlags.changeInLOS,<br>            stayRoomCount = payOrStayDetails.roomCount * multiplier,<br>            stayGrossRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount) * multiplier,<br>            stayNetRevenue = getRevenue(payOrStayDetails<br>              .copy(baseAmount = getNetRate(payOrStayDetails.netRate.toDouble, payOrStayDetails.baseAmount.toDouble)),<br>              payOrStayDetails.roomCount) * multiplier,<br>            totalStayGrossRevenue = reservationFields.totalGrossRevenue * multiplier,<br>            totalStayNetRevenue = reservationFields.totalNetRevenue * multiplier)<br><br>      case Pay =&gt;<br>        log.info(&quot;Fetching Pay Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing pay rec&quot;)<br>        resEventPayStayDetails<br>          .copy(noShowPayRooms = conFlags.noShowPayRooms,<br>            noShowPayRevenue = conFlags.noShowPayRevenue,<br>            sameDayCheckIn = conFlags.sameDayCheckIn,<br>            earlyCheckout = conFlags.earlyCheckout,<br>            unexpectedStay = conFlags.unexpectedStay,<br>            changeInLOS = conFlags.changeInLOS,<br>            payRoomCount = payOrStayDetails.roomCount * multiplier,<br>            payGrossRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount) * multiplier,<br>            payNetRevenue = getRevenue(payOrStayDetails<br>              .copy(baseAmount = getNetRate(payOrStayDetails.netRate.toDouble, payOrStayDetails.baseAmount.toDouble)),<br>              payOrStayDetails.roomCount) * multiplier,<br>            totalPayGrossRevenue = reservationFields.totalGrossRevenue * multiplier,<br>            totalPayNetRevenue = reservationFields.totalNetRevenue * multiplier)<br><br>      /*case CancelStay =&gt; if (conFlags.noShowRooms.get != 0)<br>        resEventPayStayDetails<br>          .copy(noShowRooms = Some(0),<br>            noShowRevenue = Some(0.0))<br>      else<br>        resEventPayStayDetails<br>          .copy(cancelledStayRoomCount = payOrStayDetails.roomCount * multiplier,<br>            cancelledStayRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount) * multiplier)<br>      case CancelPay =&gt; if (conFlags.noShowRooms.get != 0)<br>        resEventPayStayDetails<br>          .copy(noShowPayRooms = Some(0),<br>            noShowPayRevenue = Some(0.0))<br>      else<br>        resEventPayStayDetails<br>          .copy(cancelledPayRoomCount = payOrStayDetails.roomCount * multiplier,<br>            cancelledPayRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount) * multiplier)<br>      case CompRoom =&gt;<br>        resEventPayStayDetails.copy(compRoomCount = compRoomCnt * multiplier)*/<br>      case CancelStay =&gt;<br>        log.info(&quot;Fetching CancelStay Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing cancel stay rec&quot;)<br>        if (conFlags.noShowRooms.get == 0)<br>          resEventPayStayDetails<br>            .copy(cancelledStayRoomCount = payOrStayDetails.roomCount,<br>              cancelledStayRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount))<br>        else resEventPayStayDetails<br><br>      case CancelPay =&gt;<br>        log.info(&quot;Fetching CancelPay Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing cancel pay rec&quot;)<br>        if (conFlags.noShowPayRooms.get == 0)<br>          resEventPayStayDetails<br>            .copy(cancelledPayRoomCount = payOrStayDetails.roomCount,<br>              cancelledPayRevenue = getRevenue(payOrStayDetails, payOrStayDetails.roomCount))<br>        else resEventPayStayDetails<br><br>      case CompRoom =&gt;<br>        log.info(&quot;Fetching CompRoom Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing comp room rec&quot;)<br>        resEventPayStayDetails.copy(compRoomCount = compRoomCnt * multiplier)<br><br>      case CancelCompRoom =&gt;<br>        log.info(&quot;Fetching CancelCompRoom Record Details - generatePayStayRecords()&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        //println(&quot;5 Aug - gen pay stay | processing cancel comp room rec&quot;)<br>        resEventPayStayDetails.copy(compRoomCount = compRoomCnt * multiplier,<br>          cancelledStayRoomCount = compRoomCnt,<br>          cancelledPayRoomCount = compRoomCnt)<br><br>    }<br>  }<br><br>  /**<br>   * Get LOS between the end date and start date<br>   *<br>   * @param startDate<br>   * @param endDate<br>   * @return<br>   */<br>  /*def getLOS(startDate: Date, endDate: Date): Int = {<br>    val startDt = LocalDate.parse(startDate.toString)<br>    val endDt = LocalDate.parse(endDate.toString)<br>    DAYS.between(startDt, endDt).toInt<br>  }*/<br><br>  def getRevenue(payStayDetails: PayStayDetailsWOS3, roomCount: Int): Double = {<br><br>    log.info(&quot;Fetching the Revenue - getRevenue()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val result = (payStayDetails.baseAmount / Math.pow(10, payStayDetails.decimalPlaces) *<br>      roomCount *<br>      getLOS(payStayDetails.startDate, payStayDetails.endDate)).toDouble<br><br>    //println(&quot;5 Aug - get revenue | revenue &quot; + result)<br>    result<br>  }<br><br>  /**<br>   * Checks if net rate is not available then use the gross rate as net rate.<br>   *<br>   * @param netRate<br>   * @param baseAmount<br>   * @return<br>   */<br>  def getNetRate(netRate: Double, baseAmount: Double): Double = {<br><br>    log.info(&quot;Fetching the Net Rate - getNetRate()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    log.debug(&quot;getNetRate() | NetRate:: &quot; + netRate + &quot;  BaseAmount:: &quot; + baseAmount,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val result = if (netRate == 0.0)<br>      baseAmount<br>    else<br>      netRate<br><br>    //println(&quot;5 Aug - get net rate | net rate &quot; + result)<br>    result<br>  }<br><br>  /**<br>   * Converts the Event JSON String to Reservation Collation Event object containing both event info and payload.<br>   *<br>   * @param entity - Event JSON String<br>   * @return - ReservationCollationTriggerEvent<br>   */<br>  /*def convertToEntity(entity: String)<br>                     (implicit kafkaTopicMap: Option[Map[String, String]]): ReservationCollationTriggerEvent = {<br>    try {<br>      ReservationCollationTriggerEvent(eventInfoParser(entity), payLoadParser(entity)).validateEvent<br>    }<br>    catch {<br>      case formatError: InvalidFormatException =&gt;<br>        logger.error(&quot;ReservationEventJob ::: convertToEntity() :: InvalidFormatException : &quot;, formatError<br>          .printStackTrace())<br>        val badEntity = DefaultEventInfo(eventSource = EventSourceValue, eventIdentifier = EventIdentifierValue,<br>          recordNum = EmptyString, metadata = Map(), error = ErrorTrue, retryCounter = RetryCounterValue,<br>          destinationTopic = kafkaTopicMap.get(SourceTopic),<br>          timeStamp = System.currentTimeMillis().toString,<br>          errorType = Seq(IrrecoverableError), message = Seq(&quot;InvalidFormatException Error in Reservation Event Job -&quot; +<br>            &quot; convertToEntity() &quot;),<br>          warning = Seq())<br>        val payload = ReservationCollationPayload(payLoad = entity, payLoadType = PayLoadTypeValue)<br>        val badEvent = ReservationCollationTriggerEvent(badEntity, payload)<br>        logger.error(&quot;bad event: &quot; + badEvent.toString)<br>        badEvent<br>      case error: Exception =&gt;<br>        logger.error(&quot;ReservationEventJob ::: convertToEntity() :: Exception : &quot;, error.printStackTrace())<br>        val badEntity = DefaultEventInfo(eventSource = EventSourceValue, eventIdentifier = EventIdentifierValue,<br>          recordNum = EmptyString, metadata = Map(), error = ErrorTrue, retryCounter = RetryCounterValue,<br>          destinationTopic = kafkaTopicMap.get(SourceTopic),<br>          timeStamp = System.currentTimeMillis().toString,<br>          errorType = Seq(IrrecoverableError), message = Seq(error.getMessage), warning = Seq(ExceptionUtils.getMessage(error)))<br>        val payload = ReservationCollationPayload(payLoad = entity, payLoadType = PayLoadTypeValue, resCollationEntity = None)<br>        val badEvent = ReservationCollationTriggerEvent(badEntity, payload)<br>        logger.error(&quot;bad event: &quot; + badEvent.toString)<br>        badEvent<br>    }<br>  }*/<br><br>  /**<br>   * Parses the Event Info using JSON Parser<br>   *<br>   * @param entity - Event JSON String<br>   * @return DefaultEventInfo<br>   */<br>  /* def eventInfoParser(entity: String): DefaultEventInfo = {<br>     logger.debug(&quot;Reservation Event - Default Event Info parsed&quot;)<br>     JsonUtil.fromJson[DefaultEventInfo](entity)<br>   }*/<br><br>  /**<br>   * Parses the Payload in the Event JSON using JSON parser<br>   *<br>   * @param entity - Event JSON String<br>   * @return ReservationCollationPayload<br>   */<br>  /*  def payLoadParser(entity: String): ReservationCollationPayload = {<br>      logger.debug(&quot;Reservation Event - Payload parsed&quot;)<br>      val payLoadJson = JsonUtil.fromJson[ReservationCollationPayload](entity)<br>      ReservationCollationPayload(payLoadJson.payLoad, payLoadJson.payLoadType,<br>        Some(JsonUtil.fromJson[ReservationCollationEntity](payLoadJson.payLoad)))<br>    }*/<br><br>  /**<br>   * Get the couchbase Spark Configuration object.<br>   *<br>   * @param couchDbDetails  : couchbase details<br>   * @param couchBucketName : couchbase bucket name<br>   * @return : saprk configuration<br>   */<br>  /*def getCouchSparkConf(couchDbDetails: CouchDbDetails, couchBucketName: String): SparkConf = {<br>    val nodeDetail: List[String] = couchDbDetails.nodes<br>    val userName: String = couchDbDetails.username<br>    val password: String = couchDbDetails.password<br>    val bucketPassword = couchDbDetails.bucketpassword<br>    val kvTimeout = couchDbDetails.kvtimeout<br>    val connectionTimeout = couchDbDetails.connectiontimeout<br>    val socketTimeout = couchDbDetails.socketconnect<br><br><br>    new SparkConf()<br>      .set(UserNameConf, userName)<br>      .set(PasswordConf, password)<br>      .set(BucketNameConf + couchBucketName, bucketPassword)<br>      .set(KVTimeout, kvTimeout)<br>      .set(ConnectionTimeout, connectionTimeout)<br>      .set(SocketTimeout, socketTimeout)<br>      .set(NodeConf, nodeDetail.mkString(Semicolon))<br>  }*/<br><br>}<br><br><br><br>ReservationEventsCBHelper<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.utils.JsonUtil<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.SparkConf<br>import org.mongodb.scala.bson.{BsonArray, BsonDocument, BsonString, Document}<br><br>import java.sql.Timestamp<br>import java.time.format.DateTimeFormatter<br>import java.time.LocalDate<br>import java.util.concurrent.TimeUnit<br>import scala.collection.convert.ImplicitConversions.`iterable AsScalaIterable`<br><br>object ReservationEventsCBHelper extends Serializable with ReservationEventHelper {<br><br>  override val log = new CustomLogger(this.getClass.getName)<br><br>  /**<br>   * Retrieve the Modified Reservation with CB Rate Indicators.<br>   */<br>  def getModifiedReservationWithRateIndicators(resCollationRecords: Iterator[ReservationEventsGroupEntity],connectionConfigDetails: ConnectionConfigDetails, jobConfigDetails: JobConfigurationDetails)<br>                                              (implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails): List[ReservationCollationEntity] = {<br><br>    log.info(&quot;Fetching the Modified Reservation Details with Rate Indicators - getModifiedReservationWithRateIndicators()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val resCollationRecordList = resCollationRecords.toList<br>      .map(resRecEvent =&gt; resRecEvent.reservationCollationEvent)<br><br>    val finalRatePlanTuple = resCollationRecords<br>      .flatMap(resRecord =&gt; {<br>        val oldRes: Option[HotelReservationWOS3] = if (resRecord.reservationCollationEvent.reservations.get.oldReservation.isDefined)<br>          resRecord.reservationCollationEvent.reservations.get.oldReservation<br>        else None<br><br>        val newRes = if (resRecord.reservationCollationEvent.reservations.get.newReservation.isDefined)<br>          resRecord.reservationCollationEvent.reservations.get.newReservation<br>        else None<br><br>        val finalRes = newRes match {<br>          case Some(newReservation) =&gt;<br>            oldRes match {<br>              case Some(oldReservation) =&gt;<br>                if (Option(oldReservation).isDefined) {<br>                  Seq(oldReservation, newReservation)<br>                } else {<br>                  Seq(newReservation)<br>                }<br><br>              case None =&gt; Seq(newReservation)<br>            }<br>          case None =&gt; Seq()<br>        }<br><br>        val ratePlanTuple = getRatePlanLastUpdTsTuple(finalRes)<br><br><br>        ratePlanTuple<br>      }).toList<br><br><br>    val propCodeRatePlanLastUpdTimeMap = finalRatePlanTuple<br>      .groupBy(record =&gt; record._1)<br>      .mapValues(valueRec =&gt; valueRec.map(record =&gt; record._2))<br><br>    //Get the N1QL Query filters having the property and rateplancode like filters<br>    //  val ratePlanListFilters = getN1QLQueryFilters(finalRatePlanTuple)<br>    //    to vhk strintto timmestp format<br><br>    // TO-DO Remove Couchbase Details<br>//    val ratePlanListFilters = getN1QLQueryFilters1(finalRatePlanTuple)<br>    val filter = generateBsonFilter(finalRatePlanTuple)<br><br>    //Execute the N1QLQuery and fetch the result.<br><br>    //To-DO Remove Couchbase call<br>//    val ratePlanDocResult = getN1QLQueryResult(ratePlanListFilters,connectionConfigDetails,jobConfigDetails)<br>    val ratePlanDocResult: Array[Document] = queryRatesPlanFromDocumentDb(filter, jobConfigDetails)<br><br>    log.info(&quot; - getModifiedReservationWithRateIndicators() | Final Rate Plan Result after application of N1QLQuery filters: &quot; + ratePlanDocResult,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    //Retrieve the Couchbase indicators in a map with key as propcode_rateplan and value as the CapsRateIndicators.<br>    //    String to timestamp conversion--<br>    val modifiedResCollationInputWithRateIndDS = getCBRateIndicatorsMap(propCodeRatePlanLastUpdTimeMap, ratePlanDocResult)<br><br>    log.info(&quot; - getModifiedReservationWithRateIndicators() | Modified Reservation Details with Rate Indicators: &quot; +<br>      getModifiedOldnNewReservation(resCollationRecordList, modifiedResCollationInputWithRateIndDS),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    getModifiedOldnNewReservation(resCollationRecordList, modifiedResCollationInputWithRateIndDS)<br>  }<br><br>  /**<br>   * Get the tuple (PropCode_RatePlan,lastUpdTs).<br>   */<br><br>  // to check   String to date or date to string and hotel or cstomr reservation<br>  def getRatePlanLastUpdTsTuple(finalRes: Seq[HotelReservationWOS3]): Seq[(String, Timestamp)] = {<br><br>    log.info(&quot;Fetching the Rateplan and last updated time Details of the provided Reservation Event - getRatePlanLastUpdTsTuple()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    finalRes.flatMap(resRecord =&gt; {<br>      val propertyCode = resRecord.hotels.head.propertyCode<br>      val lastUpdTs = new java.sql.Timestamp(timestampFormat.parse(resRecord.dateTimeStamp.get.lastModify.get).getTime)<br><br>      resRecord<br>        .segments.head.offer.get.productUses<br>        .map(productUse =&gt; (propertyCode + &quot;_&quot; + productUse.ratePlanCode, lastUpdTs))<br>    }).distinct<br>  }<br><br>  /**<br>   * Get the N1QLQuery Filters.<br>   */<br>/*  def getN1QLQueryFilters(finalRatePlanTuple: List[(String, Timestamp)]): String = {<br><br>    log.info(&quot;Fetching the N1QLQuery filters for provided Rate Plan - getN1QLQueryFilters()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    finalRatePlanTuple<br>      .map(record =&gt; record._1)<br>      .distinct<br>      .map(key =&gt; {<br>        val splitKey = key.split(&quot;_&quot;)<br>        &quot;(META().id LIKE &#39;&quot; + &quot;RPGMDEFNDOC_%&#39;&quot; + &quot; and `property`.`code`=&quot; + &quot;&#39;&quot; + splitKey(0) + &quot;&#39; and &quot; + &quot;`rpgm`.`code`=&quot; + &quot;&#39;&quot; +<br>          splitKey(1) + &quot;&#39;&quot; + &quot;)&quot;<br>      })<br>      .mkString(&quot; or &quot;)<br>  }*/<br><br>  /**<br>   * Modify N1QL query<br>   */<br><br> /*   @Deprecated<br>  def getN1QLQueryFilters1(finalRatePlanTuple: List[(String, Timestamp)]): String = {<br><br>    log.info(&quot;Fetching the modified N1QLQuery Filters for provided Rate plan - getN1QLQueryFilters1()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val quote = s&quot;${&#39;&quot;&#39;}&quot;<br>    val x = finalRatePlanTuple<br>      .map(record =&gt; record._1)<br>      .distinct<br>      .map(record =&gt; {<br>        val propRpgm = record.split(&quot;_&quot;)<br>        (propRpgm(0), quote + propRpgm(1) + quote)<br>      }).groupBy(key =&gt; key._1).mapValues(_.map(_._2))<br><br>    val y = x.map(t =&gt; {<br><br>      &quot;SPLIT(META().id, \&quot;_\&quot;)[0]=&quot; + &quot;RPGMDEFNDOC&quot; + &quot; and `property`.`code`=&quot; + &quot;&#39;&quot; + t._1 + &quot;&#39; and &quot; + &quot;`rpgm`.`code`&quot; + &quot; IN &quot; + &quot;[&quot; + t._2.mkString(&quot;,&quot;) + &quot;])&quot;<br>    })<br><br>    y.mkString(&quot; or &quot;)<br><br>  }*/<br><br>  def generateBsonFilter(input: List[(String, Timestamp)]): Document = {<br>    log.info(&quot;Fetching the modified Bson Filters for provided Rate plan - generateBsonFilter()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    if(input.size&gt;0){<br>      val Rates = input<br>        .map(record =&gt; record._1)<br>        .distinct<br>        .map(record =&gt; {<br>          val propRpgm = record.split(&quot;_&quot;)<br>          (propRpgm(0), propRpgm(1))<br>        })<br>        .groupBy(_._1)<br>        .mapValues(_.map(_._2).toList)<br><br>      val bsonArray = BsonArray(Rates.map { case (property, rpgm) =&gt;<br>        BsonDocument(<br>          &quot;$and&quot; -&gt; BsonArray(<br>            BsonDocument(&quot;property.code&quot; -&gt; BsonString(property)),<br>            BsonDocument(&quot;rpgm.code&quot; -&gt; BsonDocument(&quot;$in&quot; -&gt; BsonArray(rpgm.map(BsonString(_))))<br>            )<br>          ))<br>      })<br><br>      val bsonFilter = BsonDocument(&quot;$or&quot; -&gt; bsonArray)<br>      Document(bsonFilter.toJson())<br>    }else Document()<br><br>  }<br><br>  /**<br>   * Get the Result of the N1QLQuery Result.<br>   */<br>/*    @Deprecated<br>  def getN1QLQueryResult(queryRatePlanList: String,connectionConfigDetails: ConnectionConfigDetails, jobConfigDetails: JobConfigurationDetails)<br>                        (implicit implicitCouchDetails: ImplicitCouchDetails): N1qlQueryResult = {<br><br><br>    log.info(&quot;Fetching the N1QLQuery Result after applying the provided filters to Rate Plan - getN1QLQueryResult()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val cbRatePlanIndQuery = implicitCouchDetails<br>      .n1QlQuery<br>//      .replace(&quot;bucketName&quot;, implicitCouchDetails.bucketName)TODO fix bucket name from job config<br>      .replace(&quot;bucketName&quot;,&quot;RMS-Rates&quot;)<br>      .replace(&quot;queryRatePlanList&quot;, queryRatePlanList)<br><br>    val bucket = CouchbaseConnection()<br>//      .bucket(CouchbaseConfig(getCouchSparkConf(connectionConfigDetails.couchbaseconfig.get, implicitCouchDetails.bucketName)),<br>      .bucket(CouchbaseConfig(getCouchSparkConf(connectionConfigDetails.couchbaseconfig.get, jobConfigDetails.doucmentdbmap.get(RatePlanDatabaseNameMapKey))),<br>//        implicitCouchDetails.bucketName)<br>        &quot;RMS-Rates&quot;)<br>    log.info(&quot; - getN1QLQueryResult() | Final Query cbRatePlanIndQuery: &quot; + cbRatePlanIndQuery,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br><br>    //  val adsf = bucket.query(N1qlQuery.simple(cbRatePlanIndQuery), 1000l, TimeUnit.DAYS)<br>    val adsf = bucket.query(N1qlQuery.simple(cbRatePlanIndQuery), 300000, TimeUnit.MILLISECONDS)<br><br>    log.info(&quot; - getN1QLQueryResult() | Final Result after N1QLQuery application to the Rate Plan: &quot; + adsf,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    adsf<br><br>  }*/<br><br>  def queryRatesPlanFromDocumentDb(filter: Document, jobConfigDetails: JobConfigurationDetails)(implicit conf: SparkConf): Array[Document] = {<br><br>    log.info(&quot;Fetching the Result after applying the provided filters to Rate Plan - queryDocumentDb()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    var documentsArrayWithRenamedId: Array[Document] = Array.empty[Document]<br>    // Return an empty Array if empty filter is passed<br>    if (filter.isEmpty) {<br>      documentsArrayWithRenamedId<br>    }<br>    else {<br>      val mongodbClient = getMongoClient(conf, DocumentDBOutput)<br>      val collection = mongodbClient.getDatabase(jobConfigDetails.doucmentdbmap.get(RatePlanDatabaseNameMapKey)).getCollection(jobConfigDetails.doucmentdbmap.get(RatePlanCollectionNameMapKey))<br><br>      val result = collection.find(filter)<br><br>      val documentsArray = result.toIterable.toArray<br><br>      documentsArrayWithRenamedId = documentsArray.map { document =&gt;<br>        val modifiedDocument = Document()<br>        for (entry &lt;- document.entrySet()) {<br>          val fieldName = entry.getKey()<br>          val fieldValue = entry.getValue()<br>          if (fieldName == &quot;_id&quot;) {<br>            modifiedDocument.put(&quot;documentId&quot;, fieldValue)<br>          } else {<br>            modifiedDocument.put(fieldName, fieldValue)<br>          }<br>        }<br>        modifiedDocument<br>      }.toArray<br>      documentsArrayWithRenamedId<br>    }<br>  }<br><br>  /**<br>   * Get the map of (PropCode_RatePlan_lastUpdTS , CBIndicators).<br>   */<br>  def getCBRateIndicatorsMap(propCodeRatePlanLastUpdTimeMap: Map[String, List[Timestamp]],<br>//                             ratePlanDocResult: N1qlQueryResult): Map[String, CapsAndRestrictionWOS3] = {<br>                             ratePlanDocResult: Array[Document]): Map[String, CapsAndRestrictionWOS3] = {<br><br>    log.info(&quot;Fetching the Map of Rate Plan Last Updated Time and CBIndicators - getCBRateIndicatorsMap()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val capsAndRestrictionIndMap = propCodeRatePlanLastUpdTimeMap.flatMap(<br>      lastUpdTimeTuple =&gt; {<br>        lastUpdTimeTuple._2.flatMap(<br>          ratePlanLastUpdTime =&gt; {<br>//            ratePlanDocResult.allRows.toArray()<br>            ratePlanDocResult<br>              .filter(cbDocRow =&gt; {<br>                val rateProgramDocument = JsonUtil.fromJson[RateProgramDocument](cbDocRow.toJson())<br>//val rateProgramDocument = JsonUtil.fromJson[RateProgramDocument](cbDocRow.toString)<br>                val metaId = rateProgramDocument.documentId<br>                val key = metaId.split(&quot;_&quot;)<br><br>                val (cbPropCode, cbRatePlanCode, cbStartDate, cbEndDate) =<br>                  key.length match {<br>                    case 7 =&gt; (key(1), key(2), key(5), key(6))<br>                    case 6 =&gt; (key(1), key(2), key(5), null)<br>                  }<br><br>                val docKey = cbPropCode + &quot;_&quot; + cbRatePlanCode<br>                if (docKey.equalsIgnoreCase(lastUpdTimeTuple._1)) {<br>                  val resLastUpdateTime = ratePlanLastUpdTime.toLocalDateTime.toLocalDate<br>                  log.info(&quot; - getCBRateIndicatorsMap() | CB start date:: &quot; + cbStartDate + &quot;   CB end date:: &quot; + cbEndDate,<br>                    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>                  val startDate = convertStringToLocalDate(cbStartDate, CBDateFormat)<br>                  log.info(&quot; - getCBRateIndicatorsMap() | Formatted start date:: &quot; + startDate,<br>                    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>                  if (Option(cbEndDate).isDefined) {<br>                    val endDate = convertStringToLocalDate(cbEndDate, CBDateFormat)<br>                    log.info(&quot; - getCBRateIndicatorsMap() | Formatted end date:: &quot; + endDate,<br>                      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>                    (resLastUpdateTime.isEqual(endDate) || resLastUpdateTime.isBefore(endDate)) &amp;&amp;<br>                      (resLastUpdateTime.isAfter(startDate) || resLastUpdateTime.isEqual(startDate))<br>                  } else {<br>                    resLastUpdateTime.isAfter(startDate) || resLastUpdateTime.isEqual(startDate)<br>                  }<br>                } else {<br>                  false<br>                }<br>              }).map(cbDocRow =&gt; {<br>                log.info(&quot; - getCBRateIndicatorsMap() | cbDocRow:::CAPS Indicators fetched :: &quot; + cbDocRow,<br>                  CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>                val rateProgramDocument = JsonUtil.fromJson[RateProgramDocument](cbDocRow.toString)<br>                val metaId = rateProgramDocument.documentId<br>                val capsAndRstr = rateProgramDocument.capsAndRstr<br>//                val rateOffer = rateProgramDocument.rateOffer<br>                val capsAndRestriction = CapsAndRestrictionWOS3(<br>                  guarAvail = capsAndRstr.guarAvail.map(_.toString)<br>                )<br><br>                val key = metaId.split(&quot;_&quot;)<br>                val cbPropCode = key(1)<br>                val cbRatePlanCode = key(2)<br>                val docKey = cbPropCode + &quot;_&quot; + cbRatePlanCode + &quot;_&quot; + ratePlanLastUpdTime<br>                (docKey, capsAndRestriction)<br>              })<br>          })<br>      })<br><br>    log.info(&quot; - getCBRateIndicatorsMap() | Map of Rate Plan Last Updated Time and CBIndicators: &quot; + capsAndRestrictionIndMap,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    capsAndRestrictionIndMap<br>  }<br><br>  def getModifiedOldnNewReservation(resCollationRecordList: List[ReservationCollationEntity],<br>                                    cbIndicatorsMap: Map[String, CapsAndRestrictionWOS3])<br>  : List[ReservationCollationEntity] = {<br><br>    log.info(&quot;Fetching the List of Old and New Modified Reservations - getModifiedOldnNewReservation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val resCollationRecordModified = resCollationRecordList.map(<br>      resRecord =&gt; {<br>        val resKey = resRecord.reservationKey<br>        log.info(&quot; - getModifiedOldnNewReservation() | Value of Reservation Key: &quot; + resKey,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>        val newRes = resRecord.reservations.get.newReservation<br>        val oldRes = resRecord.reservations.get.oldReservation<br><br>        val modifiedOldRes = getModifiedResWithCbRateInd(oldRes, cbIndicatorsMap)<br>        val modifiedNewRes = getModifiedResWithCbRateInd(newRes, cbIndicatorsMap)<br><br>        val modifiedOldAndNewRes = OldAndNewReservation(<br>          oldReservation = modifiedOldRes,<br>          newReservation = modifiedNewRes)<br><br>        val modifiedResCollationEntity = ReservationCollationEntity(<br>          reservationKey = resKey,<br>          reservations = Some(modifiedOldAndNewRes))<br><br>        /*     val payload = ReservationCollationPayload(<br>               payLoad = resRecord._2.payLoad,<br>               payLoadType = resRecord._2.payLoadType,<br>               resCollationEntity = Some(modifiedResCollationEntity))*/<br><br>        /* val modifiedResCollation = ReservationCollationTriggerEvent(<br>           reservationCollationEventInfo = resRecord._1,<br>           reservationCollationPayload = payload)<br> */<br>        log.info(&quot; - getModifiedOldnNewReservation() | Modified Reservation Details: &quot; + modifiedResCollationEntity,<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>        modifiedResCollationEntity<br>      })<br><br>    log.info(&quot; - getModifiedOldnNewReservation() | List of Old and New Modified Reservations: &quot; + resCollationRecordModified,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    resCollationRecordModified<br>  }<br><br>  /**<br>   * Get Modified Reservation with CB Rate Indicators.<br>   */<br>  def getModifiedResWithCbRateInd(reservation: Option[HotelReservationWOS3],<br>                                  cbIndicatorsMap: Map[String, CapsAndRestrictionWOS3]): Option[HotelReservationWOS3] = {<br><br>    log.info(&quot;Fetching the Modified Reservation with CBRate Indicators - getModifiedResWithCbRateInd()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    reservation match {<br>      case Some(resRecord) =&gt;<br>        if (Option(resRecord).isDefined) {<br>          val propCode = resRecord.hotels.head.propertyCode<br>          val lastUpdTs = resRecord.dateTimeStamp.get.lastModify<br><br>          val ratePlanMap = resRecord.ratePlans.map(ratePlan =&gt; (ratePlan.code -&gt; ratePlan.packageProducts.get.head.priceGridCode)).toMap<br>          val pmsMap = resRecord.ratePlans.map(ratePlan =&gt; (ratePlan.code -&gt; ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;))).toMap<br><br>          val modifiedSegments = resRecord<br>            .segments.head.offer.get.productUses<br>            .map(productUse =&gt; {<br>              val ratePlan = productUse.ratePlanCode<br>              //found priceplan<br>              val pricePlan = ratePlanMap(ratePlan.get)<br>              val docKey = propCode + &quot;_&quot; + ratePlan + &quot;_&quot; + lastUpdTs<br>              val checkTransientRateProgram = isRateProgramTransient(pmsMap(ratePlan.get))<br><br>              val segmentCapsIndicators = if (cbIndicatorsMap.get(docKey).isDefined)<br>                cbIndicatorsMap(docKey)<br>              else if (checkTransientRateProgram)<br>                getTransientDefaultRateIndicators<br>              else<br>                getGroupReservationRateIndicators<br><br>              val dailyRates = productUse.productRates.dailyRates.map(dailyRate =&gt; {<br><br>              val payDetailsWithCapsNRestriction = if (dailyRate.payDetails.isDefined) {<br>                Some(dailyRate<br>                .payDetails.get<br>                .copy(capsAndRestriction = segmentCapsIndicators, priceGridCode = pricePlan))<br>              } else None<br>                //added priceGridCode from rateplan<br><br>              val stayDetailsWithCapsNRestriction = if (dailyRate.stayDetails.isDefined) {<br>                Some(dailyRate.stayDetails.get.copy(capsAndRestriction = segmentCapsIndicators,<br>                  priceGridCode = pricePlan)<br>                  //added priceGridCode from rateplan<br>                )<br>              } else None<br>                dailyRate<br>                .copy(payDetails = payDetailsWithCapsNRestriction)<br>                .copy(stayDetails = stayDetailsWithCapsNRestriction)<br>              })<br>              productUse<br>                .copy(capsAndRestriction = Some(segmentCapsIndicators),<br>                  productRates = productUse.productRates.copy(dailyRates = dailyRates))<br>            })<br><br>          val modifiedPublishRes = resRecord<br>            .copy(segments = Seq(resRecord.segments.head<br>              .copy(offer = Some(resRecord.segments.head.offer.get.copy(productUses = modifiedSegments)))))<br>          //to check on this<br>          Some(modifiedPublishRes)<br>        } else None<br>      case None =&gt; None<br>    }<br>  }<br><br>  /**<br>   * Check if the Rate program is Transient or not.<br>   */<br>  def isRateProgramTransient(marketSegment: String): Boolean = {<br><br>    log.info(&quot;Checking whether the Rate Program is Transient - isRateProgramTransient()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val mrktseg = marketSegment.toLowerCase()<br>    if (contractMarketSegments.contains(mrktseg)) {<br>      log.info(&quot;Rate Program is Contract - isRateProgramTransient()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      false<br>    } else if (groupMarketSegments.contains(mrktseg)) {<br>      log.info(&quot;Rate Program is Group - isRateProgramTransient()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      false<br>    } else if (transientMarketSegments.contains(mrktseg)) {<br>      log.info(&quot;Rate Program is Transient - isRateProgramTransient()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      true<br>    } else if (totalTransientMarketSegments.contains(mrktseg)) {<br>      log.info(&quot;Rate Program is Total Transient - isRateProgramTransient()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      false<br>    } else {<br>      log.info(&quot;Rate Program is neither Transient, Contract nor Group - isRateProgramTransient()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      false<br>    }<br>  }<br><br>  /**<br>   * Transient Reservation - DEfault Rate indicator values.<br>   */<br>  def getTransientDefaultRateIndicators: CapsAndRestrictionWOS3 = {<br><br>    log.info(&quot;Retrieving the Default Rate Indicator Values of Transient Reservation - getTransientDefaultRateIndicators()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    CapsAndRestrictionWOS3(<br>//      allowCaps = Some(True),<br>//      allowRstr = Some(True),<br>      guarAvail = Some(True))<br>//      mirrorInvRstr = Some(False),<br>//      rateType = Some(&quot;REGULAR&quot;))<br>  }<br><br>  /**<br>   * Group Reservation - Rate Indicators will return default value of &quot;NA&quot;<br>   */<br>  def getGroupReservationRateIndicators: CapsAndRestrictionWOS3 = {<br><br>    log.info(&quot;Retrieving the Default Rate Indicator Values of Group Reservation - getGroupReservationRateIndicators()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    CapsAndRestrictionWOS3(<br>//      allowCaps = Some(Na),<br>//      allowRstr = Some(Na),<br>      guarAvail = Some(Na))<br>//      mirrorInvRstr = Some(Na),<br>//      rateType = Some(&quot;GROUP&quot;))<br>  }<br><br>  /**<br>   * Converts the String to Local Date.<br>   *<br>   * @param dateStr<br>   * @return<br>   */<br>  def convertStringToLocalDate(dateStr: String, dateFormat: String): LocalDate = {<br><br>    log.info(&quot;Converting Date from String Format to the desired format provided - convertStringToLocalDate()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val format = DateTimeFormatter.ofPattern(dateFormat)<br>    val lastUpdated: LocalDate = LocalDate.parse(dateStr, format)<br>    log.info(&quot; - convertStringToLocalDate() | Formatted Date: &quot; + lastUpdated,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    lastUpdated<br>  }<br><br>}<br><br><br><br>ReservationFiltrationHelper<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.fasterxml.jackson.databind.DeserializationFeature<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.EmptyString<br>import com.marriott.elevate.spark.framework.entity.DefaultEventInfo<br>import com.marriott.elevate.spark.framework.utils.{JsonUtil, S3Util}<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion.exceptions.{DataException, S3BucketException}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions.{col, lit, struct}<br><br>import java.time.format.DateTimeFormatter<br>import java.time.{LocalDate, LocalDateTime}<br><br>trait ReservationFiltrationHelper extends Serializable with ReservationIngestionHelper {<br><br><br>  val log: CustomLogger<br><br>  def reservationFiltration(data: DataFrame)(implicit jobConfigDetails: JobConfigurationDetails,<br>                                             spark: SparkSession, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationCollationEvent], DataFrame, Option[DataFrame]) = {<br><br>    import spark.implicits._<br>    /**<br>     * Dropping PII information and attributes using  convertToEntity Method and Convert Dataframe in HotelReservation Entity mentioned<br>     * for reservation filtration job<br>     */<br><br>    val firstFilter = filterPIIandMandatoryFields(data)<br><br>    /**<br>     * Filter Dataframe based on unique property code<br>     * for reservation filtration job<br>     */<br><br>    val (mgpPropertyDf, nonConvertedEntitiesDF) = filterMgpPropertyCodeDf(firstFilter)<br><br>    /**<br>     * Write Data into s3 bucket<br>     * for reservation filtration job<br>     */<br><br>    val recoverableFlag = writeToS3(mgpPropertyDf)<br>    /**<br>     * Extracting a DataFrame with specific columns of a nested DataFrame with the help of case classes<br>     * S3 Only attributes are removed and Dataframe will be input<br>     * for reservation collation job<br>     */<br>    val subsetDf = droppingS3OnlyAttribute(mgpPropertyDf)<br><br>    val nonConvertedEntitiesFinalDS = subsetDf.filter(entity =&gt; !entity._2).map(entity =&gt; entity._3)<br>    val subsetFinalDF = subsetDf.filter(entity =&gt; entity._2).map(entity =&gt; entity._1)<br><br>    val nonConvertedEntitiesFinalDF = nonConvertedEntitiesFinalDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br>    val nonConvertedEntitiesUnionDF = nonConvertedEntitiesDF.union(nonConvertedEntitiesFinalDF)<br>    val recoverableS3DF = mgpPropertyDf.<br>      toDF().<br>      select(struct(&quot;*&quot;).<br>        alias(&quot;value&quot;))<br>    val recoverableDF: Option[DataFrame] = if (recoverableFlag)<br>      Some(recoverableS3DF)<br>    else<br>      None<br>    (subsetFinalDF, nonConvertedEntitiesUnionDF, recoverableDF)<br>  }<br>  def convertToEntity(entity: String)<br>                     (implicit jobConfigDetails: JobConfigurationDetails): (Option[HotelReservation], Option[ReservationCollationEvent], String) = {<br>    try {JsonUtil.mapper<br>        .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true)<br>      JsonUtil.mapper<br>        .configure(DeserializationFeature.FAIL_ON_NULL_CREATOR_PROPERTIES, true)<br>      val data = Some(JsonUtil.fromJson[HotelReservation](entity))<br>      log.info(&quot;ConvertedtoEntity-suceess - KafkaStreamingJob ::: getEventFromString() :: &quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          appNameForLogger,<br>          jobIdentifier,<br>          jobName,<br>          customData))<br>      (data, None, entity)<br>    } catch { case exception: Exception =&gt;<br>        log.error(&quot;KafkaStreamingJob ::: getEventFromString() :: Exception :&quot;,<br>          CustomloggerMetadata(&quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            appNameForLogger,<br>            jobIdentifier,<br>            jobName,<br>            customData))<br>        new DataException(exception,<br>          CustomloggerMetadata(&quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            appNameForLogger,<br>            jobIdentifier,<br>            jobName,<br>            customData)).handle()<br>        (None, Some(ReservationCollationEvent(DefaultEventInfo(&quot;&quot;, &quot;Rn&quot;, &quot;&quot;, &quot;&quot;, &quot;&quot;, null, &quot;&quot;, null, 0, &quot;&quot;, null, null),<br>          ReservationCollationPayload(entity))), entity)<br>    }<br>  }<br>  def convertToFinalEntity(entity: String): (ReservationCollationEvent, Boolean, String) = {<br>    (ReservationCollationEvent(<br>      DefaultEventInfo(<br>        EventVersion,<br>        EventSourceValue,<br>        EventIdentifierValue,<br>        RecordNumberValue,<br>        LocalDateTime.now().toString,<br>        Map[String, String](),<br>        EmptyString,<br>        Seq[String](),<br>        RetryCounterValue,<br>        DestinationTopicValue,<br>        Seq[String](),<br>        Seq[String]()),<br>      ReservationCollationPayload(entity, HotelReservation = Some(JsonUtil.fromJson[HotelReservationWOS3](entity)))),<br>      true, entity)<br>  }<br>  def filterPIIandMandatoryFields(data: DataFrame)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails): Dataset[(Option[HotelReservation], Option[ReservationCollationEvent], String)] = {<br>    import sparkSession.implicits._<br>    data.map(x =&gt; convertToEntity(x.getString(0)))}<br>  def filterMgpPropertyCodeDf(reservationDataset: Dataset[(Option[HotelReservation], Option[ReservationCollationEvent], String)])<br>                             (implicit jobConfigDetails: JobConfigurationDetails, sparkSession: SparkSession): (DataFrame, DataFrame) = {<br>    import sparkSession.implicits._<br><br>    val irrecoverableDS = reservationDataset.filter(entity =&gt; entity._1.isEmpty || {<br>      val sleepingRoomSegments = entity._1.get.segments.filter(_.sleepingRoomInd)<br>      sleepingRoomSegments.isEmpty || sleepingRoomSegments.length &gt; 1<br>    }<br>    ).map(entity =&gt; entity._3)<br><br>    val parsedDS = reservationDataset.filter(entity =&gt; entity._1.nonEmpty &amp;&amp; entity._1.get.sourceCRSEnum.trim.toLowerCase == &quot;acrs&quot; &amp;&amp; {<br>        val sleepingRoomSegments = entity._1.get.segments.filter(_.sleepingRoomInd)<br>        sleepingRoomSegments.length == 1<br>      })<br>      .map { entity =&gt;<br>        val sleepingRoomSegment = entity._1.get.segments.filter(_.sleepingRoomInd).head<br>        val updatedHotelReservation = entity._1.get.copy(segments = Seq(sleepingRoomSegment))<br>        (Option(updatedHotelReservation), entity._2,entity._3)<br>      }<br><br>    val filteredReservations = parsedDS.filter( reservation =&gt; checkValidRatePlans(reservation)).map(entity =&gt; (entity._1, entity._2))<br><br>    val filteredIrrRecReservations = parsedDS.filter( reservation =&gt; !(checkValidRatePlans(reservation)))<br><br>    val nonConvertedEntitiesDF = irrecoverableDS.union(filteredIrrRecReservations.map(entity =&gt; entity._3)).toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val reservationAndIrrDf = filteredReservations.toDF(&quot;value&quot;, &quot;col2&quot;)<br>    val reservationDf = reservationAndIrrDf.select(&quot;value&quot;)<br><br>    log.info(&quot;DataFrame to Entity Conversion&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;,<br>        &quot;spanId&quot;,<br>        appNameForLogger,<br>        jobConfigDetails.jobidentifier,<br>        jobConfigDetails.jobname,<br>        customData))<br><br>    val column: Column = col(&quot;value.hotels&quot;)(0)<br>      .getItem(&quot;propertyCode&quot;)<br>      .getItem(&quot;code&quot;)<br>      .alias(&quot;property-code&quot;)<br>    (FilterPropertyCodeUtil.filterPropertyCode(reservationDf, column)._1, nonConvertedEntitiesDF)<br>  }<br>  def writeToS3(mgpPropertyDf: DataFrame)(implicit connectionConfigDetails: ConnectionConfigDetails, jobConfigDetails: JobConfigurationDetails): Boolean = {<br>    try {<br>      val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br>      val currentDate = LocalDate.now.format(DateTimeFormatter.ofPattern(&quot;yyyy-MM-dd&quot;))<br>      val partitionedDf = mgpPropertyDf.withColumn(&quot;date&quot;, lit(currentDate))<br>      val datePartitionCheckpoint = s&quot;$s3bucket/$jobName/${jobConfigDetails.checkpointdirectory.get(ReservationS3backup)}&quot;<br>      partitionedDf.writeStream<br>        .format(Json)<br>        .option(&quot;path&quot;, datePartitionCheckpoint)<br>        .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3bucket, jobName, jobConfigDetails.checkpointdirectory.get(ReservationFiltrationCheckpointDir)))<br>        .partitionBy(&quot;date&quot;)<br>        .start()<br>      log.info(&quot;Writen To S3&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          appNameForLogger,<br>          jobConfigDetails.jobidentifier,<br>          jobConfigDetails.jobname,<br>          customData))<br>      false<br>    }<br>    catch {<br>      case exception: Exception =&gt; log.error(&quot;Error while writing to S3 in filtration :: exception :&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            appNameForLogger,<br>            jobConfigDetails.jobidentifier,<br>            jobConfigDetails.jobname,<br>            customData))<br>        new S3BucketException(exception,<br>          CustomloggerMetadata(&quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            appNameForLogger,<br>            jobIdentifier,<br>            jobName,<br>            customData)).handle()<br>        true<br>    }<br>  }<br><br>  def droppingS3OnlyAttribute(mgpPropertyDf: DataFrame)(implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails): Dataset[(ReservationCollationEvent, Boolean, String)] = {<br>    import sparkSession.implicits._<br>    mgpPropertyDf.toJSON.map(row =&gt; if (row != null) {<br>      convertToFinalEntity(row.substring(9, row.length - 1))<br>    }<br>    else null)<br>  }<br><br>  def checkValidRatePlans(reservation: (Option[HotelReservation], Option[ReservationCollationEvent], String)): Boolean = {<br>    val ratePlanCodes = reservation._1.get.ratePlans.map(_.code)<br>    val packageProductsMap = reservation._1.get.ratePlans.map(rp =&gt; (rp.code, rp.packageProducts)).toMap<br>    val checkRatePlan = reservation._1.get.segments.head.offer.get.productUses.map { productUse =&gt;<br>      ratePlanCodes.contains(productUse.ratePlanCode.get) &amp;&amp; packageProductsMap(productUse.ratePlanCode.get).isDefined<br>    }<br>    if (checkRatePlan.contains(false)) {<br>      false<br>    } else {<br>      true<br>    }<br>  }<br>}<br><br><br><br>ReservationIngestionHelper<br><br><br>package com.marriott.mgpapp.spark.reservation.ingestion.helper<br><br>import com.marriott.elevate.spark.framework.config.{ DocumentDbDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants._<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br><br>import java.sql.Timestamp<br>import java.text.SimpleDateFormat<br>import java.time.ZoneId<br>import java.time.format.DateTimeFormatter<br>//import com.marriott.rm.platform.util.{InputType, StreamProducerHelper}<br>import com.marriott.utility.CustomLogger<br>import com.mongodb.client.{MongoClient, MongoClients}<br>import com.mongodb.connection.{SocketSettings, SslSettings}<br>import com.mongodb.{Block, ConnectionString, MongoClientSettings}<br>import com.typesafe.scalalogging.LazyLogging<br>import org.apache.spark.SparkConf<br>import org.apache.spark.sql._<br><br>import java.security.SecureRandom<br>import java.security.cert.X509Certificate<br>import java.sql.Date<br>import java.time.LocalDate<br>import java.time.temporal.ChronoUnit.DAYS<br>import java.util.concurrent.TimeUnit<br>import javax.net.ssl.{SSLContext, X509TrustManager}<br><br>trait ReservationIngestionHelper extends LazyLogging with Serializable {<br><br>  val jobIdentifier: String = &quot;ReservationIngestion&quot;<br>  val jobName: String = &quot;Reservation&quot;<br>  val log = new CustomLogger(this.getClass.getName)<br><br>//  def writeErrorToKafka(stateOutputErrorCUDEventDS: Dataset[ReservationCollationEvent],<br>//                        irrecoverableCheckpoint: String, recoverableCheckpoint: String,<br>//                        s3BucketName: String,jobConfigDetails: JobConfigurationDetails,<br>//                        sparkSession: SparkSession,  connectionConfigDetails: ConnectionConfigDetails)<br>//                       (implicit jobConfigurationDetails: JobConfigurationDetails): (StreamingQuery, StreamingQuery) = {<br>//<br>//    val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>//<br>//    val checkPointDirectory: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br>//<br>//    import sparkSession.implicits._<br>//    val server = connectionConfigDetails.brokerconfig.get.bootstrapserver<br>//    val recoverableTopic = jobConfigurationDetails.kafkatopicmap.get(RecoverableErrorTopic)<br>//    val irrecoverableTopic = jobConfigurationDetails.kafkatopicmap.get(IrrecoverableErrorTopic)<br>//    val irrecoverableDS = stateOutputErrorCUDEventDS.filter(event =&gt; event.getEventInfo.errorType.contains(IrrecoverableError)).map(event =&gt; JsonUtil.toJson(event))<br>//      .selectExpr(&quot;cast(value as String)&quot;).as[String]<br>//<br>//    val irrecoverableStream = irrecoverableDS.writeStream.format(&quot;kafka&quot;)<br>//      .outputMode(&quot;update&quot;).option(&quot;topic&quot;, irrecoverableTopic).option(&quot;kafka.bootstrap.servers&quot;, server)<br>//      .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory(irrecoverableCheckpoint)))<br>//      .option(&quot;kafka.max.request.size&quot;, 20971520)<br>//      .options(KafkaSSLOptions.addOptions(connectionConfigDetails))<br>//      .trigger(Trigger.ProcessingTime(triggerInterval))<br>//      .start()<br>//<br>//    // Write manage state - recoverable errors to kafka<br>//    val recoverableDS = stateOutputErrorCUDEventDS.filter(event =&gt; event.getEventInfo.errorType.contains(RecoverableError)).map(event =&gt; JsonUtil.toJson(event))<br>//<br>//    val recoverableStream = recoverableDS.selectExpr(&quot;cast(value as String)&quot;).as[String].writeStream.format(&quot;kafka&quot;)<br>//      .outputMode(&quot;update&quot;).option(&quot;topic&quot;, recoverableTopic).option(&quot;kafka.bootstrap.servers&quot;, server)<br>//      .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory(recoverableCheckpoint)))<br>//      .option(&quot;kafka.max.request.size&quot;, 20971520)<br>//      .options(KafkaSSLOptions.addOptions(connectionConfigDetails))<br>//      .trigger(Trigger.ProcessingTime(triggerInterval))<br>//      .start()<br>//    (irrecoverableStream, recoverableStream)<br>//  }<br><br>  def bifurcateNonCompNCompProductUses(reservation: ReservationCollationEvent) = {<br>    val segment = reservation.reservationPayload.HotelReservation.get<br>      .segments.head<br>    val productUses = segment.offer.get.productUses<br>    val ratePlanMap = reservation.reservationPayload.HotelReservation.get.ratePlans.map(ratePlan =&gt; (ratePlan.code-&gt; ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;))).toMap<br><br>    val nonComp = productUses.filterNot(productUse =&gt; isCompProductUse(ratePlanMap(productUse.ratePlanCode.get),productUse.quantity))<br>    val comp = productUses.filter(productUse =&gt; isCompProductUse(ratePlanMap(productUse.ratePlanCode.get),productUse.quantity))<br><br>    //    (segment.copy(offer = Some(segment.offer.get.copy(productUses = nonComp))),<br>    //      segment.copy(offer = Some(segment.offer.get.copy(productUses = comp))))<br>    (nonComp, comp)<br><br>  }<br><br>  def bifurcateNonCompNCompProductUsesForEvent(reservation: HotelReservationWOS3) = {<br>    val segment = reservation.segments.head<br>    val productUses = segment.offer.get.productUses<br>    val ratePlanMap = reservation.ratePlans.map(ratePlan =&gt; (ratePlan.code-&gt; ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;))).toMap<br><br>    val nonComp = productUses.filterNot(productUse =&gt; isCompProductUse(ratePlanMap(productUse.ratePlanCode.get),productUse.quantity))<br>    val comp = productUses.filter(productUse =&gt; isCompProductUse(ratePlanMap(productUse.ratePlanCode.get),productUse.quantity))<br><br>    //    (segment.copy(offer = Some(segment.offer.get.copy(productUses = nonComp))),<br>    //      segment.copy(offer = Some(segment.offer.get.copy(productUses = comp))))<br>    (nonComp, comp)<br>  }<br><br>  def isCompProductUse(propertyMarketSegment: String, roomCount: Int): Boolean = {<br>    val compRoom = propertyMarketSegment match {<br>      case marketSegment if marketSegment.toLowerCase() equals &quot;complimentary&quot; =&gt;<br>        if (roomCount &gt; 0)<br>          true<br>        else<br>          false<br>      case _ =&gt; false<br>    }<br>    compRoom<br>  }<br><br><br>  def getCompRoomCount(mktSegment: String, roomCount: Int): Int = {<br><br>    log.info(&quot;Fetching the room count based on the market code - getCompRoomCount()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    log.debug(&quot;Function - getCompRoomCount() | Value of property market Segment =&gt; &quot;<br>      + mktSegment, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    log.debug(&quot;Function - getCompRoomCount() | Value of roomCount =&gt; &quot;<br>      + roomCount, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br><br>    val compRoomCount = mktSegment match {<br>          case marketCode if marketCode.toLowerCase() equals &quot;complimentary&quot; =&gt;<br>            //logger.debug(&quot;Integer Prefix Cod &quot; + marketCode + &quot; roomCount ::&quot; + roomCount)<br>            log.debug(&quot;Function - getCompRoomCount() | Value of property market Segment =&gt; &quot;<br>              + (marketCode + &quot; roomCount ::&quot; + roomCount), CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>            if (roomCount &gt; 0)<br>              roomCount<br>            else<br>              0<br>          case _ =&gt; 0<br>        }<br>    compRoomCount<br>  }<br><br>  def getLOS(startDate: String, endDate: String): Int = {<br>    val startDt = LocalDate.parse(startDate, dateFormatter)<br>    val endDt = LocalDate.parse(endDate, dateFormatter)<br>    DAYS.between(startDt, endDt).toInt<br>  }<br><br>  def convertStringToDate(input: String): LocalDate = {<br>    LocalDate.parse(input, dateFormatter)<br>  }<br><br>  def convertStringToTimestamp(dateStr: String): Timestamp = {<br>    val parsedDate = timestampFormatter.parse(dateStr)<br>    new Timestamp(parsedDate.getTime)<br>  }<br><br><br>  def getDocumentDbSparkConf(documentDbDetails: DocumentDbDetails): SparkConf = {<br>    val userName: String = documentDbDetails.username<br>    val hostName: List[String] = documentDbDetails.host<br>    val password: String = documentDbDetails.password<br>    val connectionTimeout = documentDbDetails.connectiontimeout<br>    val documentDBName = documentDbDetails.db<br>    val documentCollectionName = documentDbDetails.collection<br>    val host = hostName.mkString(&quot;;&quot;)<br>    val uri = s&quot;mongodb://${userName}:${password}@${host}&quot;<br>    new SparkConf()<br>      .set(DocumentDbInputUri, uri)<br>      .set(DocumentDbOutputUri, uri)<br>      .set(DocumentDbInputDatabase, documentDBName)<br>      .set(DocumentDbInputCollection, documentCollectionName)<br>      .set(DocumentDbOutputDatabase, documentDBName)<br>      .set(DocumentDbOutputCollection, documentCollectionName)<br>      .set(DocumentDbReplacement, &quot;false&quot;) // Enable upsert behavior<br>      .set(DocumentDbConnectionTimeout, connectionTimeout)<br>  }<br><br>  /**<br>   * Get the MongoDB client with the specified Spark configuration<br>   *<br>   * @param conf : SparkConf containing the MongoDB URI and action referring either &quot;input&quot;/&quot;ouput&quot; as a string to pick host<br>   * @return MongoClient instance<br>   */<br>  def getMongoClient(conf: SparkConf, action: String): MongoClient = {<br>    // Extract the MongoDB connection string from the Spark configuration<br>    val connectionString = conf.get(s&quot;spark.mongodb.$action.uri&quot;) + &quot;/?retryWrites=false&amp;readpreference=primaryPreferred&quot;<br><br>    // Create a TrustManager that allows all certificates<br>    val trustManager = new X509TrustManager {<br>      override def checkClientTrusted(chain: Array[X509Certificate], authType: String): Unit = {}<br><br>      override def checkServerTrusted(chain: Array[X509Certificate], authType: String): Unit = {}<br><br>      override def getAcceptedIssuers: Array[X509Certificate] = Array.empty<br>    }<br><br>    // Create an SSLContext and initialize it with the TrustManager<br>    val sslContext = SSLContext.getInstance(&quot;TLS&quot;)<br>    sslContext.init(null, Array(trustManager), new SecureRandom())<br><br>    // Create the SSLSettings block<br>    val sslSettingsBlock = new Block[SslSettings.Builder] {<br>      override def apply(builder: SslSettings.Builder): Unit = {<br>        builder.enabled(true).invalidHostNameAllowed(true).context(sslContext)<br>      }<br>    }<br><br>    // Create a SocketSettings block to configure connection timeout<br>    val socketSettingsBlock = new Block[SocketSettings.Builder] {<br>      override def apply(builder: SocketSettings.Builder): Unit = {<br>        builder.connectTimeout(conf.get(DocumentDbConnectionTimeout).toInt, TimeUnit.MILLISECONDS) // Set the connection timeout in milliseconds (e.g., 5000 ms)<br>      }<br>    }<br><br>    // Create the MongoClientSettings with SSLSettings and connection string<br>    val settings = MongoClientSettings.builder()<br>      .applyToSslSettings(sslSettingsBlock)<br>      .applyConnectionString(new ConnectionString(connectionString))<br>      .applyToSocketSettings(socketSettingsBlock)<br>      .build()<br><br>    // Create and return the MongoClient instance<br>    val client: MongoClient = MongoClients.create(settings)<br>    client<br>  }<br><br>//  def publishToDLQ(publishDF: DataFrame, checkpointPath: Option[String], topicName: String, trigger: Trigger): Unit = {<br>//    StreamProducerHelper.writeStreamWithParams(publishDF,<br>//      topicName,<br>//      inputType = InputType.Json, checkPointPath = checkpointPath, isSingleStreaming = false<br>//    )<br>//  }<br><br>}<br><br>jobs<br><br><br>main<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.jobs<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.jobs.MgpApp<br>import org.apache.spark.SparkConf<br><br><br>object Main extends MgpApp {<br><br>  implicit val jobConfig: JobConfigurationDetails = jobConfigDetails<br><br>  implicit val connConfig: ConnectionConfigDetails = connectionConfigDetails<br><br> override def getSparkSession: Option[SparkConf] = Some(new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;ReservationIngestion&quot;).setMaster(&quot;yarn&quot;))<br> <br>  val reservationIngestionJob = new ReservationIngestionJob()<br>  reservationIngestionJob.start()<br>}<br><br>Reservationingestionjob<br><br>package com.marriott.mgpapp.spark.reservation.ingestion.jobs<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails, MessageBrokerDetails}<br>import com.marriott.elevate.spark.framework.traits.StreamingJob<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.reservation.ingestion._<br>import com.marriott.mgpapp.spark.reservation.ingestion.datasource.DocumentDBKafkaSink<br>import com.marriott.mgpapp.spark.reservation.ingestion.entity._<br>import com.marriott.mgpapp.spark.reservation.ingestion.exceptions.{DocumentDBException, KafkaException}<br>import com.marriott.mgpapp.spark.reservation.ingestion.jobs.Main.inputJsonPath<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{Deserializer, InputType, JobType, Serializer}<br>import com.marriott.rm.platform.util.{ConsumerHelperBuilder, ProducerHelperBuilder}<br>import com.marriott.mgpapp.spark.reservation.ingestion.helper.{ReservationCollationHelper, ReservationEventHelper, ReservationFiltrationHelper, ReservationIngestionHelper}<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.SparkConf<br>import org.apache.spark.sql.functions.struct<br>import org.apache.spark.sql.streaming.Trigger<br>import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}<br><br><br>class ReservationIngestionJob(implicit sparkSession: SparkSession,<br>                              jobConfigDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails)<br>  extends ReservationIngestionHelper with ReservationCollationHelper<br>    with ReservationFiltrationHelper with ReservationEventHelper<br>    with StreamingJob with Serializable {<br><br>  this: ReservationIngestionHelper =&gt;<br>  override val jobName: String = jobConfigDetails.jobname<br>  override val jobIdentifier: String = jobConfigDetails.jobidentifier<br>  override val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>  //TO-DO Remove Couchbase Details<br>  /*<br>  val couchDbDetails = connectionConfigDetails.couchbaseconfig.get<br>  implicit val conf: SparkConf = getCouchSparkConf(couchDbDetails, couchBucketName)*/<br>  override val checkPointDirectory: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br><br>  //implicit val couchBucketName: String = jobConfigDetails.couchbasemap.get(ReservationCollationBucket)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val customMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)<br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>  val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br>  var errorDs: Dataset[ReservationCollationEvent] = _<br><br>  override def process(args: Map[String, String]): Unit = {<br><br>    sys.addShutdownHook {<br>      println(&quot;Shutdown hook triggered, stopping streaming queries.&quot;)<br>      sparkSession.streams.active.foreach(query =&gt; {<br>        println(s&quot;Stopping query: ${query.name}&quot;)<br>        query.stop()<br>      })<br>    }<br><br>    /*    val bucketName: String = jobConfigDetails.couchbasemap.get(HPPRatesBucket)<br>        val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)*/<br><br>    implicit val implicitCouchDetails: ImplicitCouchDetails = ImplicitCouchDetails(&quot;RMS-RES-JOURNAL&quot;, &quot;SELECT META().id as `documentId`, `bucketName`.* FROM `bucketName` WHERE queryRatePlanList&quot;)<br><br>    //    val checkpointDirMap = jobConfigDetails.checkpointdirectory.get<br>    //    val irrecoverableErrorCheckpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(IrrecoverableErrorCheckpointDir).get))<br>    //    val recoverableErrorS3Checkpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(RecoverableErrorS3CheckpointDir).get))<br>    //    val recoverableErrorDocDBCheckpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(RecoverableErrorDocDBCheckpointDir).get))<br>    //    val trigger: Trigger = Trigger.ProcessingTime(triggerInterval)<br><br>    /**<br>     * Consuming EEH Kafka topic data<br>     * Use val file = &quot;HotelReservationV8.avsc&quot; when From_avro Function is used<br>     * Using UDF method, File is not required<br>     * */<br><br>    val data = consumeDataFromEEH(InputType.Avro)<br><br>    log.info(&quot;Read Data From EEH&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    /**<br>     * Method call to Filtration Step<br>     */<br>    val (filteredReservationDS, irrecoverableDF, recoverableS3DF) = reservationFiltration(data)<br>    //println(&quot;5 Aug - Data after filtration step - &quot;)<br>    //filteredReservationDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Reservation Filtration Success &quot;, customMetadata)<br><br>    log.info(&quot;Initiating Irrecoverable Errors messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    //    irrecoverableDF.writeStream.foreachBatch((batchDS: org.apache.spark.sql.DataFrame, _: Long) =&gt; {<br>    //<br>    //      StreamProducerHelper.writeWithParam(batchDS.toDF(),<br>    //        topic = irrecoverableTopic,<br>    //        inputType = InputType.Json, checkPointPath = irrecoverableErrorCheckpoint)<br>    //<br>    //    }).option(&quot;checkpointLocation&quot;, irrecoverableErrorCheckpoint.get)<br>    //      .option(&quot;kafka.max.request.size&quot;, 20971520)<br>    //      .trigger(trigger)<br>    //      .start()<br>    //    publishToDLQ(irrecoverableDF,irrecoverableErrorCheckpoint, irrecoverableTopic,trigger)<br><br><br>    //    recoverableS3DF match {<br>    //      case Some(value) =&gt;<br>    //        log.info(&quot;Initiating Recoverable Errors messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //        recoverableS3DF.get.writeStream.foreachBatch((batchDS: org.apache.spark.sql.DataFrame, _: Long) =&gt; {<br>    //<br>    //          StreamProducerHelper.writeWithParam(batchDS.toDF(),<br>    //            topic = recoverableTopic,<br>    //            inputType = InputType.Json, checkPointPath = recoverableErrorS3Checkpoint)<br>    //<br>    //        }).option(&quot;checkpointLocation&quot;, recoverableErrorS3Checkpoint.get)<br>    //          .option(&quot;kafka.max.request.size&quot;, 20971520)<br>    //          .trigger(trigger)<br>    //          .start()<br>    //    publishToDLQ(recoverableS3DF,recoverableErrorDocDBCheckpoint,recoverableTopic,trigger)<br><br>    //<br>    //      case None =&gt;<br>    //        log.info(&quot;No messages to publish for Recoverable Scenario of S3 write&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName,customData))<br>    //    }<br><br>    /**<br>     * Method call to Collation Step<br>     */<br>    val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession)<br>    //println(&quot;5 Aug - Data after collation step - &quot;)<br>    //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    //    filteredReservationDS.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;,false).start()<br>    //    recoverableDocDBDF.writeStream.foreachBatch((batchDS: org.apache.spark.sql.DataFrame, _: Long) =&gt; {<br>    //<br>    //      StreamProducerHelper.writeWithParam(batchDS.toDF(),<br>    //        topic = recoverableTopic,<br>    //        inputType = InputType.Json, checkPointPath = recoverableErrorDocDBCheckpoint)<br>    //<br>    //    }).option(&quot;checkpointLocation&quot;, recoverableErrorDocDBCheckpoint.get)<br>    //      .option(&quot;kafka.max.request.size&quot;, 20971520)<br>    //      .trigger(trigger)<br>    //      .start()<br>    //    publishToDLQ(recoverableDocDBDF,recoverableErrorDocDBCheckpoint,recoverableTopic,trigger)<br><br>    try {<br>      /**<br>       * Method call for event step<br>       */<br>      persistDS(newNOldRecordsDS,<br>        connectionConfigDetails.brokerconfig.get,<br>        jobConfigDetails.checkpointdirectory.get,<br>        jobConfigDetails.jobname,<br>        s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval, sparkSession, connectionConfigDetails)<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Error while writing to couchbase :: Exception :: reservationCollation():&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>        new DocumentDBException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>    }<br>    sparkSession.streams.awaitAnyTermination<br>  }<br><br>  def persistDS(newNOldRecordsDS: Dataset[ReservationEventsEntity],<br>                kafkaConfig: MessageBrokerDetails,<br>                checkpointDir: Map[String, String],<br>                jobName: String,<br>                s3bucket: String,<br>                kafkaMaxSize: Float,<br>                triggerInterval: Int,<br>                sparkSession: SparkSession,<br>                connectionConfigDetails: ConnectionConfigDetails)(implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails): Unit = {<br><br><br>    newNOldRecordsDS<br>      .writeStream<br>      .foreachBatch((batchDS: org.apache.spark.sql.Dataset[com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationEventsEntity], _: Long) =&gt; {<br><br>        batchDS.persist()<br><br>        batchDS.foreach { row =&gt;<br>          val documentDBSinkObject = new DocumentDBKafkaSink(conf, kafkaConfig, kafkaMaxSize)<br>          documentDBSinkObject.open(1.toLong, 2)<br>          documentDBSinkObject.process(row)<br>        }<br><br><br>        //println(&quot;5 Aug - Data before res event - &quot;)<br>        //batchDS.show(false)<br><br><br>        val resEventOutDS = reservationEvent(batchDS, sparkSession, jobConfigDetails, connectionConfigDetails)<br>        //val resEventOutDF = resEventOutDS.toDF()<br>        val checkPointPathKafka: Option[String] = Option(S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationEventCheckpointDir)))<br><br>        logger.info(&quot;checkpoint path:&quot; + checkPointPathKafka)<br><br>        val resEventOutFinalDF = resEventOutDS.select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>        produceDataToEEH(resEventOutFinalDF)<br>      })<br>      .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationCollationCheckpointDir)))<br>      .trigger(Trigger.ProcessingTime(triggerInterval))<br>      .start()<br><br>  }<br><br>  def produceDataToEEH(data: DataFrame)(implicit checkpointPath: Option[String]): Unit = {<br><br>    val input = InputType.Avro<br>    val checkPointPathKafka: Option[String] = checkpointPath<br>    val destinationTopic = KafkaProperties.destinationTopic<br>    val schemaPath = &quot;reservationEventSchema.avsc&quot;<br>    try {<br><br>      val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(schemaPath))<br>        .withCaseClass(Some(classOf[EEHData]))<br>        .withCheckPoint(checkPointPathKafka)<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>      log.info(&quot;pushed data into Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while pushing data into kafka topic :: Exception:&quot; + exception.printStackTrace(),<br>          customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>    }<br>  }<br><br>  def consumeDataFromEEH(inputType: InputType.Type)(implicit spark: SparkSession): DataFrame = {<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    //val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:-1, &quot;1&quot;:-1, &quot;2&quot;:-1, &quot;3&quot;:12148, &quot;4&quot;: -1, &quot;5&quot;: -1, &quot;6&quot;: -1, &quot;7&quot;: -1}}&quot;&quot;&quot;<br>    val startingOffset: String = &quot;latest&quot;<br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br><br>      readStreamData.consume().select(&quot;value&quot;)<br><br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data fram kafka topic&quot;, customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>        val emptyDataFrame: DataFrame = null<br>        emptyDataFrame<br>    }<br>  }<br><br><br>}<br><br></pre><p>Reservation full job</p><pre>===============================================build.sbt file====================<br><br>resolvers ++= Seq(<br>  Resolver.mavenLocal,<br>  &quot;Artifactory&quot; at &quot;https://artifactory.marriott.com/artifactory/&quot;,<br>  &quot;ariesrelease&quot; at &quot;https://artifactory.marriott.com/artifactory/libs-release/&quot;,<br>  &quot;ariessnapshot&quot; at &quot;https://artifactory.marriott.com/artifactory/libs-snapshot/&quot;<br>)<br><br><br>lazy val ReservationIngestion =<br>  (project in file(&quot;ReservationIngestion&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;ReservationIngestion&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.ReservationIngestion)<br>    .settings(dependencyOverrides ++= Dependencies.ReservationIngestionOverrides)<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;rm-functional-event-handler-scala&quot; % &quot;0.0.9-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;rm-shared-utils-ingestion&quot; % &quot;0.0.1-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;shared-utils-logging&quot; % &quot;0.0.5-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(coverageExcludedFiles :=<br>      &quot;src/main/scala/com/marriott/mgpapp/spark/reservation/ingestion/jobs/Main.scala&quot;)<br><br><br>      lazy val assemblySettings = assemblyMergeStrategy in assembly := {<br>  case m if m.toLowerCase.endsWith(&quot;manifest.mf&quot;) =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.matches(&quot;meta-inf.*\\.sf$&quot;) =&gt; MergeStrategy.discard<br>  case &quot;log4j.properties&quot; =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.startsWith(&quot;meta-inf/services/&quot;) =&gt; MergeStrategy.filterDistinctLines<br>  case &quot;reference.conf&quot; =&gt; MergeStrategy.concat<br>  case _ =&gt; MergeStrategy.first<br>}<br><br><br><br>======================================dependencies file=======================================<br><br><br> val sparkVersion = &quot;3.3.0&quot;<br>  val starterKitVersion_0_0_4 = &quot;0.0.7-SNAPSHOT&quot;<br><br>   val ReservationIngestion: Seq[ModuleID] = Seq(kafka,<br>    couchbaseConnector,<br>    elevateFramework_06502,<br>    //elevateFramework_06502,<br>    scalaScoverage,<br>    embeddedKafkaTest,<br>    mongoScala,<br>    mongoSpark,<br>    scalaTest_3_2_17,<br>    amazonaws,<br>    scalamock_5_1_0,<br>    scalatestplus,<br>    scalatest_1_16_42,<br>    mockitoInline<br><br>  )<br><br>===================================================main class==============================================<br>  object Main extends MgpApp {<br><br>  implicit val jobConfig: JobConfigurationDetails = jobConfigDetails<br><br>  implicit val connConfig: ConnectionConfigDetails = connectionConfigDetails<br><br> override def getSparkSession: Option[SparkConf] = Some(new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;ReservationIngestion&quot;).setMaster(&quot;local&quot;))<br> <br>  val reservationIngestionJob = new ReservationIngestionJob()<br>  reservationIngestionJob.start()<br>}<br><br>======================================================================<br><br>class ReservationIngestionJob(implicit sparkSession: SparkSession,<br>                              jobConfigDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails)<br>  extends ReservationIngestionHelper with ReservationCollationHelper<br>    with ReservationFiltrationHelper with ReservationEventHelper<br>    with StreamingJob with Serializable {<br><br><br><br>    this: ReservationIngestionHelper =&gt;<br>  override val jobName: String = jobConfigDetails.jobname<br>  override val jobIdentifier: String = jobConfigDetails.jobidentifier<br>  override val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>  //TO-DO Remove Couchbase Details<br>  /*<br>  val couchDbDetails = connectionConfigDetails.couchbaseconfig.get<br>  implicit val conf: SparkConf = getCouchSparkConf(couchDbDetails, couchBucketName)*/<br>  override val checkPointDirectory: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br><br>  //implicit val couchBucketName: String = jobConfigDetails.couchbasemap.get(ReservationCollationBucket)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val customMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)<br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>  val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br>  var errorDs: Dataset[ReservationCollationEvent] = _<br><br>  override def process(args: Map[String, String]): Unit = {<br><br>   val data = consumeDataFromEEH(InputType.Avro)<br><br>   def consumeDataFromEEH(inputType: InputType.Type)(implicit spark: SparkSession): DataFrame = { <br><br>   ---------------just info--------<br>   object InputType extends Enumeration {<br>  type Type = Value<br>  /**<br>   * For [[InputType.Avro Input Type Avro]] it will be preferred to use [[Deserializer.deserializeWithUdfForAvro Deserializer deserializeWithUdfForAvro]]<br>   * instead of [[Deserializer.deserializerWithUdf Deserializer deserializerWithUdf]].<br>   *  - [[Deserializer.deserializeWithUdfForAvro Deserializer deserializeWithUdfForAvro]]<br>   *      This provides the data in value column as nested struct value.<br>   *      This allows the user to access nested fields like dataframe.select(&quot;value.nestedFieldName&quot;)<br>   *  - [[Deserializer.deserializerWithUdf Deserializer deserializerWithUdf]]<br>   *      This provides the data in value column as json string (not struct type).<br>   *      This does not allow the user to access nested fields like dataframe.select(&quot;value.nestedFieldName&quot;).<br>   */<br>  val Avro: InputType.Value = Value<br>  val Json: InputType.Value = Value<br>  val None: InputType.Value = Value<br>}<br>-------------------info closed-----------------------------------------------------------------<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    //val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:-1, &quot;1&quot;:-1, &quot;2&quot;:61893, &quot;3&quot;:-1, &quot;4&quot;: -1, &quot;5&quot;: -1, &quot;6&quot;: -1, &quot;7&quot;: -1}}&quot;&quot;&quot;<br>   // val startingOffset: String = &quot;latest&quot;<br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br><br>        // now we get the data  in  the json string<br><br>      readStreamData.consume().select(&quot;value&quot;)<br><br><br>      -----------------------just for info---------------------------------------------<br><br>      def consume(): DataFrame = {<br><br>    val kafkaStreamDF: DataFrame = jobTypes match {<br>      case JobType.Batch =&gt;<br>        val customSpanData: CustomSpanData = new CustomSpanDataBuilder(s&quot;kafka-batch-consumer-$topic&quot;, SpanKind.CONSUMER)<br>          .withAttribute(&quot;SparkSession.appName&quot;, sparkSession.sparkContext.appName)<br>          .withAttribute(&quot;SparkSession.ApplicationId&quot;, sparkSession.sparkContext.applicationId)<br>          .build()<br>        val (df, _) = batchTelemetry.retrieveTraceInfo(readBatchJobs(), customSpanData)<br>        df<br>      case JobType.Streaming =&gt;<br>        if(KafkaProperties.streamTraceEnabled){<br>          val customSpanData: CustomSpanData = new CustomSpanDataBuilder(s&quot;kafka-stream-consumer-$topic&quot;, SpanKind.CONSUMER)<br>            .withAttribute(&quot;SparkSession.appName&quot;, sparkSession.sparkContext.appName)<br>            .withAttribute(&quot;SparkSession.ApplicationId&quot;, sparkSession.sparkContext.applicationId)<br>            .build()<br>          readStreamJobs().generateTraceForeachRow(customSpanData)<br>        } else {<br>          readStreamJobs()<br>        }<br>    }<br><br>---------------------------------info closed-----------------------------------------------------------------------------<br><br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data fram kafka topic&quot;, customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br><br> ---------------------------------------------------------------just for info---------------------------------------------------<br> override def handle(input: Map[String, Any]): Boolean = {<br>    log.error(<br>      s&quot;&quot;&quot; Exception has occurred in starter kit, error code = $getErrorCode, error message = $getErrorMessage<br>         ,error cause = ${getErrorCause.getClass.toString}&quot;&quot;&quot;.stripMargin, customMetadata)<br>    true<br>  }<br><br>}<br>---------------------------------info closed-----------------------------------------------------------------------------<br><br>  ************************************************star info, how to create a emptydataframe becuase the output demands-------------------------<br>        val emptyDataFrame: DataFrame = null<br>        emptyDataFrame<br>    }<br>  }<br><br><br><br><br><br>--------------------------------step 2 ---------------pending for later-----------------------------------------------------------<br> val (filteredReservationDS, irrecoverableDF, recoverableS3DF) = reservationFiltration(data)<br><br><br><br> ----------------------------------step 3---------------------------------------------------------------------------------<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>===============================================build.sbt file====================<br><br>resolvers ++= Seq(<br>  Resolver.mavenLocal,<br>  &quot;Artifactory&quot; at &quot;https://artifactory.marriott.com/artifactory/&quot;,<br>  &quot;ariesrelease&quot; at &quot;https://artifactory.marriott.com/artifactory/libs-release/&quot;,<br>  &quot;ariessnapshot&quot; at &quot;https://artifactory.marriott.com/artifactory/libs-snapshot/&quot;<br>)<br><br><br>lazy val ReservationIngestion =<br>  (project in file(&quot;ReservationIngestion&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;ReservationIngestion&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.ReservationIngestion)<br>    .settings(dependencyOverrides ++= Dependencies.ReservationIngestionOverrides)<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;rm-functional-event-handler-scala&quot; % &quot;0.0.9-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;rm-shared-utils-ingestion&quot; % &quot;0.0.1-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(libraryDependencies += &quot;com.marriott.mgp&quot; % &quot;shared-utils-logging&quot; % &quot;0.0.5-SNAPSHOT&quot; exclude(&quot;org.apache.logging.log4j&quot;, &quot;log4j-to-slf4j&quot;))<br>    .settings(coverageExcludedFiles :=<br>      &quot;src/main/scala/com/marriott/mgpapp/spark/reservation/ingestion/jobs/Main.scala&quot;)<br><br><br>      lazy val assemblySettings = assemblyMergeStrategy in assembly := {<br>  case m if m.toLowerCase.endsWith(&quot;manifest.mf&quot;) =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.matches(&quot;meta-inf.*\\.sf$&quot;) =&gt; MergeStrategy.discard<br>  case &quot;log4j.properties&quot; =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.startsWith(&quot;meta-inf/services/&quot;) =&gt; MergeStrategy.filterDistinctLines<br>  case &quot;reference.conf&quot; =&gt; MergeStrategy.concat<br>  case _ =&gt; MergeStrategy.first<br>}<br><br><br><br>======================================dependencies file=======================================<br><br><br> val sparkVersion = &quot;3.3.0&quot;<br>  val starterKitVersion_0_0_4 = &quot;0.0.7-SNAPSHOT&quot;<br><br>   val ReservationIngestion: Seq[ModuleID] = Seq(kafka,<br>    couchbaseConnector,<br>    elevateFramework_06502,<br>    //elevateFramework_06502,<br>    scalaScoverage,<br>    embeddedKafkaTest,<br>    mongoScala,<br>    mongoSpark,<br>    scalaTest_3_2_17,<br>    amazonaws,<br>    scalamock_5_1_0,<br>    scalatestplus,<br>    scalatest_1_16_42,<br>    mockitoInline<br><br>  )<br><br>======================main class==============================================<br>  object Main extends MgpApp {<br><br>  implicit val jobConfig: JobConfigurationDetails = jobConfigDetails<br><br>  implicit val connConfig: ConnectionConfigDetails = connectionConfigDetails<br><br> override def getSparkSession: Option[SparkConf] = Some(new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;ReservationIngestion&quot;).setMaster(&quot;local&quot;))<br> <br>  val reservationIngestionJob = new ReservationIngestionJob()<br>  reservationIngestionJob.start()<br>}<br><br>======================================================================<br><br>class ReservationIngestionJob(implicit sparkSession: SparkSession,<br>                              jobConfigDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails)<br>  extends ReservationIngestionHelper with ReservationCollationHelper<br>    with ReservationFiltrationHelper with ReservationEventHelper<br>    with StreamingJob with Serializable {<br><br><br><br>    this: ReservationIngestionHelper =&gt;<br>  override val jobName: String = jobConfigDetails.jobname<br>  override val jobIdentifier: String = jobConfigDetails.jobidentifier<br>  override val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>  //TO-DO Remove Couchbase Details<br>  /*<br>  val couchDbDetails = connectionConfigDetails.couchbaseconfig.get<br>  implicit val conf: SparkConf = getCouchSparkConf(couchDbDetails, couchBucketName)*/<br>  override val checkPointDirectory: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br><br>  //implicit val couchBucketName: String = jobConfigDetails.couchbasemap.get(ReservationCollationBucket)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val customMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)<br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>  val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br>  var errorDs: Dataset[ReservationCollationEvent] = _<br><br>  override def process(args: Map[String, String]): Unit = {<br><br>   val data = consumeDataFromEEH(InputType.Avro)<br><br>   def consumeDataFromEEH(inputType: InputType.Type)(implicit spark: SparkSession): DataFrame = { <br><br>   ---------------just info--------<br>   object InputType extends Enumeration {<br>  type Type = Value<br>  /**<br>   * For [[InputType.Avro Input Type Avro]] it will be preferred to use [[Deserializer.deserializeWithUdfForAvro Deserializer deserializeWithUdfForAvro]]<br>   * instead of [[Deserializer.deserializerWithUdf Deserializer deserializerWithUdf]].<br>   *  - [[Deserializer.deserializeWithUdfForAvro Deserializer deserializeWithUdfForAvro]]<br>   *      This provides the data in value column as nested struct value.<br>   *      This allows the user to access nested fields like dataframe.select(&quot;value.nestedFieldName&quot;)<br>   *  - [[Deserializer.deserializerWithUdf Deserializer deserializerWithUdf]]<br>   *      This provides the data in value column as json string (not struct type).<br>   *      This does not allow the user to access nested fields like dataframe.select(&quot;value.nestedFieldName&quot;).<br>   */<br>  val Avro: InputType.Value = Value<br>  val Json: InputType.Value = Value<br>  val None: InputType.Value = Value<br>}<br>-------------------info closed-----------------------------------------------------------------<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    //val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:-1, &quot;1&quot;:-1, &quot;2&quot;:61893, &quot;3&quot;:-1, &quot;4&quot;: -1, &quot;5&quot;: -1, &quot;6&quot;: -1, &quot;7&quot;: -1}}&quot;&quot;&quot;<br>   // val startingOffset: String = &quot;latest&quot;<br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br><br>        // now we get the data  in  the json string<br><br>      readStreamData.consume().select(&quot;value&quot;)<br><br><br>      -----------------------just for info---------------------------------------------<br><br>      def consume(): DataFrame = {<br><br>    val kafkaStreamDF: DataFrame = jobTypes match {<br>      case JobType.Batch =&gt;<br>        val customSpanData: CustomSpanData = new CustomSpanDataBuilder(s&quot;kafka-batch-consumer-$topic&quot;, SpanKind.CONSUMER)<br>          .withAttribute(&quot;SparkSession.appName&quot;, sparkSession.sparkContext.appName)<br>          .withAttribute(&quot;SparkSession.ApplicationId&quot;, sparkSession.sparkContext.applicationId)<br>          .build()<br>        val (df, _) = batchTelemetry.retrieveTraceInfo(readBatchJobs(), customSpanData)<br>        df<br>      case JobType.Streaming =&gt;<br>        if(KafkaProperties.streamTraceEnabled){<br>          val customSpanData: CustomSpanData = new CustomSpanDataBuilder(s&quot;kafka-stream-consumer-$topic&quot;, SpanKind.CONSUMER)<br>            .withAttribute(&quot;SparkSession.appName&quot;, sparkSession.sparkContext.appName)<br>            .withAttribute(&quot;SparkSession.ApplicationId&quot;, sparkSession.sparkContext.applicationId)<br>            .build()<br>          readStreamJobs().generateTraceForeachRow(customSpanData)<br>        } else {<br>          readStreamJobs()<br>        }<br>    }<br><br>---------------------------------info closed<br><br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data fram kafka topic&quot;, customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>        val emptyDataFrame: DataFrame = null<br>        emptyDataFrame<br>    }<br>  }</pre><p>Reservation Recoverable:</p><pre>  val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:2186, &quot;1&quot;:2313, &quot;2&quot;:2157, &quot;3&quot;:2574, &quot;4&quot;: 2168, &quot;5&quot;: 2103, &quot;6&quot;: 2682, &quot;7&quot;:2265}}&quot;&quot;&quot;<br>   // val startingOffset: String = &quot;latest&quot;<br>   <br>   <br>   <br>   <br>           .withCheckPointPath(Some(&quot;test---------------checkpoint-----------&quot;))<br>     <br>     <br>     .set(&quot;spark.yarn.am.waitTime&quot;, &quot;2400s&quot;)<br>.set(&quot;spark.sql.broadcastTimeout&quot;, &quot;1800s&quot;)<br>.set(&quot;spark.sql.adaptive.enabled&quot;, &quot;true&quot;)<br>.set(&quot;spark.sql.adaptive.coalescePartitions.enabled&quot;, &quot;true&quot;)<br>.set(<br>  &quot;spark.sql.adaptive.optimizeSkewsInRebalancePartitions.enabled&quot;,<br>  &quot;true&quot;<br>)<br><br><br><br><br>method_latest : (ds,list)------replace ----&gt; ds,boolean)<br><br>fetchpms--&gt;list(true flag)---------------&gt;(isreocverable=true)-------&gt;    true<br><br>getdocdb------&gt;tuple(_,flag)-------------&gt; (recerrorfla= true)-----&gt; output<br><br><br>upsert:   { _id: &quot;ANKIT202_2023-07-10&quot; }<br>{ _id: RegExp(&quot;ANKIT555&quot;) }<br><br><br>RATEDFENDOC_BOGAR_KAR001<br><br>{ _id: RegExp(&quot;RATEDFENDOC_BOGAR_KAR001&quot;) }<br><br><br><br>{ _id: &quot;RATEDFENDOC_BOGAR_KAR002_2024-04-30 13:07:18_&quot; }<br><br><br>{ _id: { &quot;$regex&quot;:&quot;RATEDFENDOC_BOGAR_KAR002_&quot; }}<br><br><br>_id<br>&quot;RATEDFENDOC_BOGAR_ABC123_2024-04-22 14:37:18_&quot;<br><br>{ _id: &quot;ANKIT786_2023-07-10&quot; }<br>{ _id: { &quot;$regex&quot;:&quot;ANKIT786&quot; }}<br><br>{ _id: RegExp(&quot;RATEDFENDOC_BOGAR_KAR001&quot;) }<br><br><br><br>&quot;<br>id&quot;: { &quot;$regex&quot;: &quot;^ABCD<br>&quot; }<br><br><br><br>(-------------------printing key-----------------,RATEDFENDOC_BOGAR_KAR002_)<br><br><br>(test_data-----,Document{{_id=RATEDFENDOC_BOGAR_KAR002_2024-04-30 13:07:18_, propertyCode=Document{{code=BOGAR}}, generationTime=2024-04-30T13:07:18.123Z, ratePlanDetails=[Document{{code=KAR002, details=Document{{houseUseInd=true, propertyMarketSegmentation=BR, blackoutDatesAllowedInd=true, advancePurchaseRequiredInd=true}}, inventoryTypeCodes=[code1, code2, code3, code4], sellDate=Document{{start=2023-04-01, end=2023-07-10, duration=null}}, ratePlanID=null, descriptions=[Document{{text=[Description]}}], advanceBookingRestrictions=[Document{{advancePurchaseInd=false, offset=Document{{min=0, max=999}}, effectiveDowEnum=[Monday, Wednesday]}}], associatedPriceGrids=[Document{{code=KARTIK, actionEnum=Add}}, Document{{code=03, actionEnum=Remove}}, Document{{code=04, actionEnum=NoChange}}], packageProducts=[Document{{code=packageProd, usage=usage, priceIncludedInd=true, quantity=10, priceGridCode=YELLOW, corporateEnforcedInd=true}}, Document{{code=packageProd, usage=usage, priceIncludedInd=true, quantity=10, priceGridCode=YELLOW, corporateEnforcedInd=true}}, Document{{code=packageProd, usage=usage, priceIncludedInd=false, quantity=10, priceGridCode=KAR2, corporateEnforcedInd=true}}], allocations=[Document{{timespan=Document{{start=2020-11-10, end=2020-11-30, duration=null}}, atMostNumber=78}}], stayRestriction=Document{{timespan=Document{{start=2023-04-01, end=2023-07-10, duration=null}}, lengthOfStay=[Document{{min=1, max=10}}], requiredCheckInDowEnum=[Monday, Wednesday], stayOverDowEnum=[Monday, Wednesday], noStayDowEnum=[Monday, Wednesday]}}, marketType=Document{{marketCode=Corporate}}, rateBucket=rateBucket, bestFlexibleRateInd=true}}], capsAndRstr=Document{{allowRstr=true, allowCaps=false, mirrorInvRstr=false, guarAvail=false}}, expiresAt=null}})<br><br><br><br><br>case class MainClass(<br>  generationTime: LocalDateTime,<br>  actionEnum: String,<br>  warningIds: Seq[Int],<br>  confirmationNumber: ConfirmationNumber,<br>  cancellationNumber: Option[String],<br>  bookingComIndexNumber: Option[String],<br>  creator: Creator,<br>  requestor: Requestor,<br>  confirmationIDs: Seq[ConfirmationID],<br>  cancellationIDs: Option[String],<br>  imageID: String,<br>  committedSequenceNumber: Int,<br>  from: Option[String],<br>  rewardBenefitInvokedInd: Option[String],<br>  pnr: Pnr,<br>  externalShareId: String,<br>  unsolicitedCancelId: String,<br>  externalTravelWithId: String,<br>  links: Seq[Link],<br>  emailNotifInd: Boolean,<br>  smsNotifInd: Boolean,<br>  groupContracts: Seq[GroupContract],<br>  installments: Seq[Installment],<br>  hotels: Seq[Hotel],<br>  timespan: Timespan,<br>)<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>===========================================<br> def fetchPMSfromRateplan(<br>  reservationRecordsList: List[ReservationCollationEvent]<br>)(implicit jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): List[ReservationCollationEvent] = {<br><br>  val updatedList = reservationRecordsList.map { row =&gt;<br>    val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>      val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>        if (ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;).isEmpty) { // -- if pms is empty then fetch its value from MongoDB<br>          val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>          val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) // -- 2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>          <br>          try {<br>            val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>            val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>            val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>            val key = &quot;RATEDFENDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + row.reservationPayload.HotelReservation.get.ratePlans.head.code + &quot;_&quot;<br>            val regexPattern = s&quot;&quot;&quot;^$key.*_$$&quot;&quot;&quot;.r.pattern<br>            val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br><br>            val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>            val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>            val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>            val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>            rateplanupdated<br>          } catch {<br>            case exception: Exception =&gt;<br>              new ExceptionHandler(exception).handle(<br>                Map(<br>                  jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>                  inventorytype.jobName -&gt; jobConfigDetails.jobname,<br>                  customErrorMessage -&gt; {<br>                    s&quot;&quot;&quot;connectDocDBAndFetchResult :: Error connecting to DocumentDB&quot;&quot;&quot;<br>                  }<br>                )<br>              )<br>              ratePlan // Return the original ratePlan in case of an error<br>          }<br>        } else {<br>          ratePlan<br>        }<br>      }<br>      rec.copy(ratePlans = updatedRatePlans)<br>    }<br>    row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>  }<br>  updatedList<br>}<br><br><br>============================================================<br><br> else{<br>      iterator.toList.map(grpBlockEvent =&gt; GroupBlockEntity(payload = grpBlockEvent.groupBlockPayload.payLoad,isRecoverable = true)).distinct<br>    }<br><br><br>    ===============================================<br><br>     def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br><br>//    var replacement = &quot;Corporate&quot;<br>    val updatedList = reservationRecordsList.map { row =&gt;<br>      val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>        val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>          if (ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;).isEmpty) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>            val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>            val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>            try {<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDFENDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + row.reservationPayload.HotelReservation.get.ratePlans.head.code + &quot;_&quot;<br>              val regexPattern = s&quot;&quot;&quot;^$key.*_$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br><br>              //            val filter = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KARP1_2024-04-30 13:07:18_&quot;)  // -- filter data based on id<br><br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br><br>            else<br>            {<br>              ratePlan<br>            }<br><br>          rec.copy(ratePlans = updatedRatePlans)<br><br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br><br>      }<br>        //            catch {<br>        //              case exception: Exception =&gt;<br>        //                new ExceptionHandler(exception).handle(<br>        //                  Map(<br>        //                    jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>        //                    inventorytype.jobName -&gt; jobConfigDetails.jobname,<br>        //                    customErrorMessage -&gt; {<br>        //                      s&quot;&quot;&quot;connectDocDBAndFetchResult :: Error connecting to DocumentDB&quot;&quot;&quot;<br>        //                    }<br>        //                  )<br>        //                )<br>        //                List()<br>        // ReservationCollationEvent.ReservationCollationPayload.payload<br>        // DocumentDBRecordsRecoverable(None, isRecoverable = true, row.messageGenerationTime, row.sourceEvent)<br><br>        //            }<br><br>        // iterator.toList.map(grpBlockEvent =&gt; GroupBlockEntity(payload = grpBlockEvent.groupBlockPayload.payLoad,isRecoverable = true)).distinct<br>        // ReservationCollationEvent.ReservationCollationPayload.payload<br>      // catch {is recoverable =true)<br>     // Reservarationcollationevnt(ReservationCollationEvent.ReservationCollationPayload.payload, isrecoverable= true)<br>      ReservationCollationEvent(row.reservationInfo,row.reservationPayload,isRecoverable = True)<br>    }<br>    updatedList<br>  }<br><br><br>=========================================================================<br><br>need to pass row.reservationInfo,row.reservationPayload,isRecoverable = True)<br><br>    } but it is giving error in row as row is outside the function of updatedlist give ma solution for below method <br><br>   def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]= {<br><br>    //    var replacement = &quot;Corporate&quot;<br>    Try {<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if (ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;).isEmpty) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(joConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDFENDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + row.reservationPayload.HotelReservation.get.ratePlans.head.code + &quot;_&quot;<br>              val regexPattern = s&quot;&quot;&quot;^$key.*_$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br><br>              //            val filter = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KARP1_2024-04-30 13:07:18_&quot;)  // -- filter data based on id<br><br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              ratePlan<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>            jobName -&gt; jobConfigurationDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from from ${collection.getNamespace.getFullName} for key :: $key&quot;&quot;&quot;<br>            }<br>          )<br>        )<br>        //DocumentDBRecordsRecoverable(None, isRecoverable = true, row.messageGenerationTime, row.sourceEvent)<br>        ReservationCollationEvent(row.reservationInfo,row.reservationPayload,isRecoverable = True)<br><br>    }<br>    }<br><br>    =============================================================================================================================================<br><br><br><br><br>this is the method<br> def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):<br>  List[ReservationCollationEvent]= {<br><br><br>    //    var replacement = &quot;Corporate&quot;<br>    val updatedList = reservationRecordsList.map { row =&gt;<br>      try {<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if (!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      catch<br>      {<br>        case exception: Exception =&gt;<br>          new ExceptionHandler(exception).handle(<br>            Map(<br>              jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>              jobName -&gt; jobConfigurationDetails.jobname,<br>              customErrorMessage -&gt; {<br>                s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from<br>                   |${jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;)} for reservation ID ${row.reservationInfo}&quot;&quot;&quot;.stripMargin<br>              }<br>            )<br>          )<br>          ReservationCollationEvent(row.reservationInfo, row.reservationPayload, isRecoverable = true)<br>      }<br>    }<br><br>  updatedList<br>}<br><br><br>it does not throw exception<br><br><br><br>this is enclosed in this  def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                ,connectionConfigDetails: ConnectionConfigDetails):<br>  NewNOldReservationRecordsErrorEntity = {<br><br>    log.info(&quot;Fetching the latest reservation record out of list of reservation records with same reservation key - getLatestReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val reservationRecordsList = reservationRecords.toList<br>    // need to add recoverable logic here in fetchPMSfromRateplan()<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br><br><br>    which in turn is enclosed in this<br><br>     def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    try {<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups(getLatestReservationRecord)<br>      //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>    } catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>            jobName -&gt; jobConfigDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records&quot;&quot;&quot;.stripMargin<br>            }<br>          )<br>        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br><br><br>so when fetchPMS has the exception will it return  to the catch block  of reservationCollation or do we need some change in the above approach?<br><br><br><br><br><br>solution<br><br>=====================================================================================================<br><br>Yes, you can use throw in the catch block of fetchPMSfromRateplan to propagate the exception to the caller. This will allow the exception to be caught in the reservationCollation method or any other method that calls fetchPMSfromRateplan. <br><br>Here is how you can modify fetchPMSfromRateplan to throw the exception after logging it:<br><br>def fetchPMSfromRateplan(reservationRecordsList: List[ReservationCollationEvent])<br>                        (implicit jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): List[ReservationCollationEvent] = {<br>  reservationRecordsList.map { row =&gt;<br>    try {<br>      val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>        val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>          if (!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) {<br>            val documentDbDetails = connectionConfigDetails.documentconfig.get<br>            val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>            val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>            val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>            val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>            val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>            val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>            val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>            val docdbdata = collection.find(filter).first().toJson()<br>            val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>            val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>            ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>          } else {<br>            val marketCode = ratePlan.marketType.get.marketCode<br>            ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>          }<br>        }<br>        rec.copy(ratePlans = updatedRatePlans)<br>      }<br>      row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>    } catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>            jobName -&gt; jobConfigurationDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from<br>                 |${jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;)} for reservation ID ${row.reservationInfo}&quot;&quot;&quot;.stripMargin<br>            }<br>          )<br>        )<br>        throw exception // Propagate the exception<br>    }<br>  }<br>}<br>With this change, any exception that occurs within fetchPMSfromRateplan will be rethrown after it is logged, allowing the caller to handle it.<br><br>Then, you can catch the exception in the reservationCollation method:<br><br>def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                         sparkSession: SparkSession)<br>                        (implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups(getLatestReservationRecord)<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty<br>      )<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br><br>    (newNOldRecordsDS, recoverableDocDBDF)<br>  } catch {<br>    case exception: Exception =&gt;<br>      new ExceptionHandler(exception).handle(<br>        Map(<br>          jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>          jobName -&gt; jobConfigDetails.jobname,<br>          customErrorMessage -&gt; {<br>            s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records&quot;&quot;&quot;.stripMargin<br>          }<br>        )<br>      )<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      println(&quot;checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>This way, if an exception occurs in fetchPMSfromRateplan, it will be propagated to reservationCollation, where it can be caught and handled appropriately.<br><br><br><br><br><br>==============================================================================================================================<br><br><br><br><br><br>this is the method<br> def reservationEvent(rawCollationEventDS: Dataset[ReservationEventsEntity],payloadEvent:DataFrame) (implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails,<br>                       connectionConfigDetails: ConnectionConfigDetails, conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails): Dataset[ReservationKafkaEvent] = {<br><br>    log.info(&quot;Fetching the Reservation Events Details - reservationEvent()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br><br>      this is where it gets called<br>        def persistDS(newNOldRecordsDS: Dataset[ReservationEventsEntity],<br>                payloadEvent:DataFrame,<br>                kafkaConfig: MessageBrokerDetails,<br>                checkpointDir: Map[String, String],<br>                jobName: String,<br>                s3bucket: String,<br>                kafkaMaxSize: Float,<br>                triggerInterval: Int,<br>                sparkSession: SparkSession,<br>                connectionConfigDetails: ConnectionConfigDetails)(implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails): Unit = {<br><br><br>    newNOldRecordsDS<br>      .writeStream<br>      .foreachBatch((batchDS: org.apache.spark.sql.Dataset[com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationEventsEntity], _: Long) =&gt; {<br><br>        batchDS.persist()<br><br>        batchDS.foreach { row =&gt;<br>          val documentDBSinkObject = new DocumentDBKafkaSink(conf, kafkaConfig, kafkaMaxSize)<br>          documentDBSinkObject.open(1.toLong, 2)<br>          documentDBSinkObject.process(row)<br>        }<br><br><br>        //println(&quot;5 Aug - Data before res event - &quot;)<br>        //batchDS.show(false)<br><br><br>        val resEventOutDS = reservationEvent(batchDS,payloadEvent)<br><br>        why am getting this error <br>        C:\Users\ajune750\RMMGP-29659-Reservation-6th-May-trunalimerged\dev-mgp\ReservationETL\ReservationIngestion\src\main\scala\com\marriott\mgpapp\spark\reservation\ingestion\jobs\ReservationIngestionJob.scala:185:45<br>not enough arguments for method reservationEvent: (implicit sparkSession: org.apache.spark.sql.SparkSession, implicit jobConfigDetails: com.marriott.elevate.spark.framework.config.JobConfigurationDetails, implicit connectionConfigDetails: com.marriott.elevate.spark.framework.config.ConnectionConfigDetails, implicit conf: org.apache.spark.SparkConf, implicit implicitCouchDetails: com.marriott.mgpapp.spark.reservation.ingestion.entity.ImplicitCouchDetails)org.apache.spark.sql.Dataset[com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationKafkaEvent].<br>Unspecified value parameters sparkSession, jobConfigDetails, connectionConfigDetails...<br>        val resEventOutDS = reservationEvent(batchDS,payloadEvent)<br><br><br>solve this<br><br><br><br><br><br><br><br><br><br>===========================================================================================================<br><br><br>def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                ,connectionConfigDetails: ConnectionConfigDetails):<br>  NewNOldReservationRecordsErrorEntity = {<br><br>  <br>    val reservationRecordsList = reservationRecords.toList<br>//    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br>    try {<br>      val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br><br><br>      val recoverableRecord = resRecWithCMS.filter(_.isRecoverable)<br>      val recoverableEvent = recoverableRecord.map(x =&gt; x.reservationPayload.payLoad)<br>      <br>      val (docDBReservationRecord, recoverableErrFlag) = getRecordFromDocDB(reservationKey.get)<br><br>      val uniqueReservationRecordsList = filterExistingReservationsInDocDB(<br>        resRecWithCMS,<br>        docDBReservationRecord)<br><br><br>      val sortedReservationRecordList = uniqueReservationRecordsList<br>        .sortBy(record =&gt; {<br>          (record.reservationPayload.HotelReservation<br>            .get.reservationStatusEnum match {<br>            case ReservationStatusEnum.Committed =&gt; 1<br>            case ReservationStatusEnum.Modified =&gt; 2<br>            case ReservationStatusEnum.Cancelled =&gt; 3<br>          },<br>            record.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify)<br>        })<br><br>      if (sortedReservationRecordList.nonEmpty) {<br>        val newReservation = if (sortedReservationRecordList.tail.nonEmpty) {<br>          sortedReservationRecordList.tail.foldLeft(sortedReservationRecordList.head) {<br>            (res1, res2) =&gt; {<br><br><br>              log.info(&quot; - Function - getLatestReservationRecord() | Value of Res1.lastModify =&gt; &quot; +<br><br>                res1.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify,<br>                CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>              log.info(&quot; - Function - getLatestReservationRecord() | Value of Res2.lastModify =&gt; &quot; +<br>                res2.reservationPayload.HotelReservation.get.dateTimeStamp.get.lastModify,<br>                CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>              if (res1.reservationPayload.HotelReservation.get<br>                .dateTimeStamp.get.lastModify.equals(res2.reservationPayload<br>                  .HotelReservation.get.dateTimeStamp.get.lastModify)) {<br>                //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are equal&quot;)<br>                val result = if (docDBReservationRecord.isEmpty)<br>                  getReservationEventRecords(reservationKey, res2, None).newReservation.get<br>                else<br>                  docDBReservationRecord.get<br><br>                result<br>              }<br>              else {<br>                val newNOldReservationEntity = {<br>                  if (res1.reservationPayload.HotelReservation.isDefined<br>                    &amp;&amp; res2.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>                    &amp;&amp; res1.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>                    val result = getReservationEventRecords(reservationKey, res1, Some(res2))<br>                    //println(&quot;5 Aug - getLatestReservationRecord | flow where both res lastmodify are not equal | res1 - defined, modified, res2 - committed | newNOldReservationEntity &quot; + result)<br>                    result<br>                  } else {<br>                    val result = getReservationEventRecords(reservationKey, res2, Some(res1))<br>                    result<br>                  }<br>                }<br><br>                if (newNOldReservationEntity.newReservation.get.getEventInfo.errorType.contains(IrrecoverableError)) {<br>                  return NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(reservationKey,<br>                    Some(res2),<br>                    Some(res1),<br>                    recoverableErrFlag)))<br>                } else {<br>                  newNOldReservationEntity.newReservation.get<br>                }<br>              }<br>            }<br>          }<br>        } else if (docDBReservationRecord.isDefined) {<br>          //println(&quot;5 Aug - getLatestReservationRecord | sorted list tail is empty &amp; decdb is defined | docdb record - &quot; + docDBReservationRecord)<br>          docDBReservationRecord.get<br>        } else {<br>          val newNOldReservationEntity = getReservationEventRecords(reservationKey, sortedReservationRecordList.head, None)<br>          newNOldReservationEntity.newReservation.get<br>        }<br>        val oldReservationInUniqueList = sortedReservationRecordList.tail.foldLeft(sortedReservationRecordList.head) {<br>          (res1, res2) =&gt; {<br>            // res1<br>            if (res1.reservationPayload.HotelReservation.isDefined) {<br>              val result = getReservationEventRecords(reservationKey, res1, None).newReservation.get<br>              //println(&quot;5 Aug - getLatestReservationRecord | oldReservationInUniqueList | res1 defined - &quot; + result)<br>              result<br>            } else {<br>              val result = getReservationEventRecords(reservationKey, res2, Some(res1)).oldReservation.get<br>              //println(&quot;5 Aug - getLatestReservationRecord | oldReservationInUniqueList | res1 not defined - &quot; + result)<br>              result<br>            }<br>          }<br>        }<br><br>        docDBReservationRecord match {<br>          case None =&gt;<br>            if (newReservation.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>              &amp; oldReservationInUniqueList.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>              val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>                reservationKey,<br>                Some(oldReservationInUniqueList),<br>                None, recoverableErrFlag)))<br>              //println(&quot;5 Aug - getLatestReservationRecord | docDBReservationRecord absent | new committed &amp; old modified - &quot; + result)<br>              result<br>            } else {<br>              val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>                reservationKey,<br>                Some(newReservation),<br>                None, recoverableErrFlag)))<br>              result<br>            }<br>          case Some(docDbRecord) =&gt;<br>            if (newReservation.reservationPayload.HotelReservation.get<br>              == docDbRecord.reservationPayload.HotelReservation.get) {<br>              val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>                None, None, None, recoverableErrFlag)))<br>              result<br>            } else if (newReservation.reservationPayload.HotelReservation.get.reservationStatusEnum == Committed<br>              &amp; docDbRecord.reservationPayload.HotelReservation.get.reservationStatusEnum == Modified) {<br>              val result = NewNOldReservationRecordsErrorEntity(None)<br>              result<br>            } else {<br>              val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>                reservationKey,<br>                Some(newReservation),<br>                docDBReservationRecord,<br>                recoverableErrFlag)))<br>              result<br>            }<br>        }<br>      }<br>      else {<br>        val result = NewNOldReservationRecordsErrorEntity(Some(NewNOldReservationEventErrorEntity(<br>          None, None, None, recoverableErrFlag)))<br>        //println(&quot;5 Aug - getLatestReservationRecord | sortedlist empty | result - &quot; + result)<br>        result<br>      }<br>    }catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>        throw ex // rethrow the exception to be caught by reservationCollation<br>    }<br><br>  }<br><br><br><br>  ===========================================================================================================<br><br><br><br>  def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):<br>  List[ReservationCollationEvent]= {<br><br>//val  DocumentDBInput2= &quot;asdfg&quot;<br>    //    var replacement = &quot;Corporate&quot;<br>    val updatedList = reservationRecordsList.map { row =&gt;<br>      try {<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if (!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      catch<br>      {<br>        case exception: Exception =&gt;<br>          new ExceptionHandler(exception).handle(<br>            Map(<br>              jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>              jobName -&gt; jobConfigurationDetails.jobname,<br>              customErrorMessage -&gt; {<br>                s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from<br>                   |${jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;)} for reservation ID ${row.reservationInfo}&quot;&quot;&quot;.stripMargin<br>              }<br>            )<br>          )<br>          //ReservationCollationEvent(row.reservationInfo, row.reservationPayload, isRecoverable = true)<br>          throw exception<br>      }<br>    }<br><br>  updatedList<br>}<br><br><br><br><br><br>this is not goting in the catch when it throws the error as we have used the try block after map which is an iterator <br>so we are going to use this method<br><br><br><br><br>===========================================================================================================================================================================<br>in this method in the catch block we want to throw the exception in the catch block <br> def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb: &quot; + ex.printStackTrace())<br>        List.empty[ReservationCollationEvent]<br><br>    }<br>  }<br><br><br>  ===============solution========================================<br><br>  catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb-fetchPMS: &quot; + ex.printStackTrace())<br>        //List.empty[ReservationCollationEvent]<br>        throw ex<br><br>    }<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>    ===================================================================================<br><br>    this is the method<br>      def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb-fetchPMS: &quot; + ex.printStackTrace())<br>        //List.empty[ReservationCollationEvent]<br>        throw ex<br><br>    }<br>  }<br><br><br><br><br>  it throws the exception  enclosed in catch block which is expected but <br><br><br>  it does not catch the exception in reservationCollation <br><br><br><br>this is enclosed in this  def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                ,connectionConfigDetails: ConnectionConfigDetails):<br>  NewNOldReservationRecordsErrorEntity = {<br><br>    log.info(&quot;Fetching the latest reservation record out of list of reservation records with same reservation key - getLatestReservationRecord()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val reservationRecordsList = reservationRecords.toList<br>    // need to add recoverable logic here in fetchPMSfromRateplan()<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br><br><br>    which in turn is enclosed in this<br><br>     def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    try {<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups(getLatestReservationRecord)<br>      //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>    } catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>            jobName -&gt; jobConfigDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records&quot;&quot;&quot;.stripMargin<br>            }<br>          )<br>        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br><br><br>so when fetchPMS has the exception will it return  to the catch block  of reservationCollation or do we need some change in the above approach?<br><br><br><br>===========================================================================================================================<br><br><br><br>this time it went to the catch block of  getLatestReservationRecord as this (Exception in getLatestReservationRecord:) is printed in the logs  <br>catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>        throw ex // rethrow the exception to be caught by reservationCollation<br>    }<br><br><br>but again it didnt go to the exception of <br><br><br>idealy if we get the exception in getLatestReservationRecord which is enclosed in thr try block of reservationCollation method it should have gone to catch block of  reservationCollation<br><br>what is causing the issue<br><br>def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    try {<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups(getLatestReservationRecord)<br>      //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.info(&quot;entering in the catch block of reservationCollation &quot;)<br>//        new ExceptionHandler(exception).handle(<br>//          Map(<br>//            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>//            jobName -&gt; jobConfigDetails.jobname,<br>//            customErrorMessage -&gt; {<br>//              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records-2&quot;&quot;&quot;.stripMargin<br>//            }<br>//          )<br>//        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        println(&quot;printing schema of  recoverableEvent&quot;)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br><br><br><br>  =========================================<br><br>  now  this has printed<br>  Exception in mapGroups while processing key:<br><br>  this also printed:<br><br>  println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br><br>  and this also<br><br>  log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br><br>but again it does not go under the catch block below<br><br>catch {<br>      case ex: Exception =&gt;<br>        log.info(&quot;entering in the catch block of reservationCollation &quot;)<br>//        new ExceptionHandler(exception).handle(<br>//          Map(<br>//            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>//            jobName -&gt; jobConfigDetails.jobname,<br>//            customErrorMessage -&gt; {<br>//              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records-2&quot;&quot;&quot;.stripMargin<br>//            }<br>//          )<br>//        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        println(&quot;printing schema of  recoverableEvent&quot;)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br>  ===============================================again this is not going to the catch block instead it is completing the try block and getting outside <br><br>  so i have implemented the same outside the reservation collation<br><br>    val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession)<br>   <br>    log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>    finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>    println(&quot;recoverable checkpoint&quot;)<br>    produceRecoverableData(finalDF)<br><br>  but this was therre in the catch block of reservationCollation. since it was not going there i kept it outside the reservationCollation but i would be needing a flag or catch condition otherwisse it will throw the data in this in every case but i want it only in the case of exception<br><br><br><br>  =======================================================================================================<br><br><br>  this reservationCollation is defined in reservationcollationhelper but called in reservationingestionjobs  so if (errorAccumulator.value &gt; 0)  is giving eeror as it is importe and defined in  reservationcollationhelper<br><br><br><br>  ============================<br><br>  It seems that the exceptions occurring within the Spark transformations are not propagating as expected to the driver, which is a common challenge when working with distributed computing frameworks like Spark. Instead, those exceptions are being caught or logged at the executor level and not making their way back to the driver’s catch block.<br><br>Given this scenario, you can use a flag or an accumulator to signal an error condition back to the driver. Here’s how you can implement it:<br><br>Use an Accumulator to Signal an Error:<br>Accumulators are variables that are only &quot;added&quot; to, such as counters, and can be used to signal back to the driver that an error has occurred in the executors.<br><br><br>Modify the reservationCollation Function to Use an Accumulator:<br>We will create an accumulator to indicate if an exception has been encountered.<br><br><br>Check the Accumulator After the Transformation:<br>After the transformation, we will check the accumulator&#39;s value to determine if an exception occurred and handle it accordingly.<br>Here&#39;s how you can modify your code:<br><br>Step 1: Define the Accumulator<br>import org.apache.spark.util.LongAccumulator<br><br>val errorAccumulator: LongAccumulator = sparkSession.sparkContext.longAccumulator(&quot;Error Accumulator&quot;)<br>Step 2: Modify reservationCollation to Use the Accumulator<br>def reservationCollation(<br>    rawReservationDS: Dataset[ReservationCollationEvent],<br>    sparkSession: SparkSession,<br>    errorAccumulator: LongAccumulator<br>)(implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups((key, records) =&gt; {<br>        getLatestReservationRecord(Some(key), records) match {<br>          case Right(result) =&gt; result<br>          case Left(ex) =&gt;<br>            log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>            errorAccumulator.add(1) // Signal an error<br>            throw ex<br>        }<br>      })<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;)<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;)<br>    (newNOldRecordsDS, recoverableDocDBDF)<br><br>  } catch {<br>    case exception: Exception =&gt;<br>      log.error(&quot;Exception in reservationCollation: &quot;, exception)<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      log.info(&quot;Printing schema of recoverableEvent&quot;)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      log.info(&quot;Checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>Step 3: Check the Accumulator After the Transformation<br>val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession, errorAccumulator)<br><br>if (errorAccumulator.value &gt; 0) {<br>  log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>  val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>  finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>  println(&quot;recoverable checkpoint&quot;)<br>  produceRecoverableData(finalDF)<br>}<br>Explanation:<br><br><br>Accumulator Definition:<br>The errorAccumulator is defined and initialized.<br><br><br>Using the Accumulator in reservationCollation:<br>The accumulator is passed to the reservationCollation method.<br>Inside mapGroups, when an exception occurs, the accumulator is incremented.<br><br><br>Checking the Accumulator Value:<br>After calling reservationCollation, the value of the accumulator is checked.<br>If the accumulator has a value greater than 0, it indicates that an error occurred during the transformation, and the error handling logic is executed.<br>This approach ensures that you can signal an error condition from within the distributed Spark operations back to the driver, allowing you to handle the error appropriately in the driver code.<br><br><br>Copy<br>Thank you for your feedback.<br>Icon<br><br><br><br>====================================================================================================================================================================<br><br><br>===============================================================================================================================================================================<br><br><br>getting this issue :  forward reference extends over definition of value x$2<br>    val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession,errorAccumulator)<br><br><br><br><br><br><br>ok|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||<br><br><br> I dont understand why the code is not going in getLatestReservationRecord method ,also not going in the exception  match {<br>            case Right(result) =&gt; result<br>            case Left(ex) =&gt;<br>              log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>              errorAccumulator.add(1)<br>              throw ex<br>          }<br>        }) as well it directly goes out from this line when it enters collation why??<br><br><br> def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession, errorAccumulator: LongAccumulator<br>  )(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br><br><br>    //val errorAccumulator: LongAccumulator = sparkSession.sparkContext.longAccumulator(&quot;Error Accumulator&quot;)<br>   // try {<br><br>    //changing from here<br>    println(&quot;stop_1&quot;)<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups((key, records) =&gt; {<br>          getLatestReservationRecord((key), records) match {<br>            case Right(result) =&gt; result<br>            case Left(ex) =&gt;<br>              log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>              errorAccumulator.add(1)<br>              throw ex<br>          }<br>        })<br><br><br><br>      println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br><br><br><br>      ================answer----------------------------------<br><br>      The issue you are encountering seems to be related to how the code is handling the transformation and grouping operations within the reservationCollation method. Specifically, the problem might be occurring within the mapGroups transformation.<br><br>Here&#39;s a detailed breakdown of possible reasons why the code might not be entering the getLatestReservationRecord method or the exception match block, and directly moving out from the collation:<br><br>GroupByKey and MapGroups Execution: The groupByKey and mapGroups transformations are lazy operations in Spark. They will not be executed until an action is called on the resulting dataset. Ensure that there is an action (like count, collect, write, etc.) being called on newNOldRecordsDS or recoverableDocDBDF after the transformations. If there is no action, Spark will not execute the transformations.<br><br><br><br>====================adjust this method like this when it throws exception it will get outside the reservationCollation and then outside the   reservationCollation method this should be get under this  if (errorAccumulator.value &gt; 0) {<br>      log.info(&quot;accumulator is greater than 0&quot;)<br>      val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      println(&quot;recoverable checkpoint&quot;)<br>      produceRecoverableData(finalDF)<br><br><br><br>def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession, errorAccumulator: LongAccumulator<br>  )(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br><br>//    try {<br>//      val latestReservationRecordDS = rawReservationDS<br>//        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>//        .mapGroups(getLatestReservationRecord)<br>//    try {<br>//      val latestReservationRecordDS = rawReservationDS<br>//        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>//        .mapGroups((key, records) =&gt; {<br>//          try {<br>//            getLatestReservationRecord((key),records)<br>//          } catch {<br>//            case ex: Exception =&gt;<br>//              log.error(&quot;Exception in mapGroups while processing key: &quot; + key, ex)<br>//              throw ex<br>//          }<br>//        })<br><br>    //val errorAccumulator: LongAccumulator = sparkSession.sparkContext.longAccumulator(&quot;Error Accumulator&quot;)<br>   // try {<br><br>    //changing from here<br>    println(&quot;stop_1&quot;)<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups((key, records) =&gt; {<br>          getLatestReservationRecord((key), records) match {<br>            case Right(result) =&gt; result<br>            case Left(ex) =&gt;<br>              log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>              errorAccumulator.add(1)<br>              throw ex<br>          }<br>        })<br><br><br><br>      println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>//    } catch {<br>//      case ex: Exception =&gt;<br>//        log.info(&quot;entering in the catch block of reservationCollation &quot;)<br>////        new ExceptionHandler(exception).handle(<br>////          Map(<br>////            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>////            jobName -&gt; jobConfigDetails.jobname,<br>////            customErrorMessage -&gt; {<br>////              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records-2&quot;&quot;&quot;.stripMargin<br>////            }<br>////          )<br>////        )<br>//        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>//        println(&quot;printing schema of  recoverableEvent&quot;)<br>//        recoverableEvent.printSchema()<br>//        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>//        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>//        println(&quot;recoverable checkpoint&quot;)<br>//        produceRecoverableData(finalDF)<br>//        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>//        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br>//<br>//        (emptyDataset, emptyDataFrame)<br>//<br>//<br>//    }<br>  }<br><br>  the main issue we are excpecting that it will thro w exception from  getLatestReservationRecord((key), records) match {<br>            case Right(result) =&gt; result<br>            case Left(ex) =&gt;<br>              log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>              errorAccumulator.add(1)<br>              throw ex<br>          }<br>        }) and this exception will get aught outside the reservationcollation method  and based on any simple condition it it gets the error it will be given to the above code for outside <br><br>        change the accumulator or give any other simple approach or do anything i just want the results<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>=============================================================================================================<br><br>        remove this accumuulator logic and catch the exception for both the persistDS as well as well as reservation collation in on catch block <br><br>  try{<br>    val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession,errorAccumulator)<br>    //println(&quot;5 Aug - Data after collation step - &quot;)<br>    //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>//    println(&quot;printing schema of  recoverableEvent&quot;)<br>//    recoverableEvent.printSchema()<br>    println(&quot;errorAccumulator value is &quot; , errorAccumulator )<br>    if (errorAccumulator.value &gt; 0) {<br>      log.info(&quot;accumulator is greater than 0&quot;)<br>      val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      println(&quot;recoverable checkpoint&quot;)<br>      produceRecoverableData(finalDF)<br>    }<br><br><br>   // try {<br>      /**<br>       * Method call for event step<br>       */<br>      persistDS(newNOldRecordsDS,<br>        payloadEvent,<br>        connectionConfigDetails.brokerconfig.get,<br>        jobConfigDetails.checkpointdirectory.get,<br>        jobConfigDetails.jobname,<br>        s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval)<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Error while writing to couchbase :: Exception :: reservationCollation():&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>        new DocumentDBException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>    }<br>    sparkSession.streams.awaitAnyTermination<br>  }<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>  ===============================================================restarting==================================================================================================<br><br>  this is the method  which we deliberatlely giving the wrong password  so that it fails and goes to the  catch exception, the exception we are getting is this<br><br>     def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br>      println(&quot;stop_3&quot;)<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            println(&quot;yaha par aayega&quot;)<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              println(&quot;yaha par nahi aana chahiye&quot;)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      println(&quot;Exiting fetchPMSfromRateplan normally&quot;)<br>      updatedList<br>    } catch {<br><br>      case ex: Exception =&gt;<br>        println(&quot;Exception caught in fetchPMSfromRateplan&quot;)<br>        log.error(&quot;Exception while fetching from docdb-fetchPMS: &quot; + ex.printStackTrace())<br>        //List.empty[ReservationCollationEvent]<br>        println(&quot;stop_4&quot;)<br>        throw ex<br><br>    }<br>  }<br>  once fetchPMSfromRateplan will throw exception it will go under this method getLatestReservationRecord<br><br>  def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                 ,connectionConfigDetails: ConnectionConfigDetails): NewNOldReservationRecordsErrorEntity = {<br><br>    val reservationRecordsList = reservationRecords.toList<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br>        val (docDBReservationRecord, recoverableErrFlag) = getRecordFromDocDB(reservationKey.get)<br><br><br><br>this methos is called from <br><br>  def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession, errorAccumulator: LongAccumulator<br>  )(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity]<br>    //, DataFrame<br>    ) = {<br>println(&quot;print11----&quot;)<br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>   <br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>    println(&quot;stop_1&quot;)<br><br>val latestReservationRecordDS = rawReservationDS<br>  .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>  .mapGroups(getLatestReservationRecord)<br><br><br><br>we want the exception thrown from fetchPMS method will get into the catch block of the below<br> override def process(args: Map[String, String]): Unit = {<br>  try{<br>    val (newNOldRecordsDS) = reservationCollation(filteredReservationDS, sparkSession,errorAccumulator)<br>      persistDS(newNOldRecordsDS,<br>        payloadEvent,<br>        connectionConfigDetails.brokerconfig.get,<br>        jobConfigDetails.checkpointdirectory.get,<br>        jobConfigDetails.jobname,<br>        s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval)<br>    }<br>    catch {<br><br>      case exception: Exception =&gt;<br>        println(&quot;is it entering here-------------------------&quot;)<br>        val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;recoverable checkpoint&quot;)<br>        produceRecoverableData(finalDF)   }<br>    sparkSession.streams.awaitAnyTermination<br>  }<br><br><br>it goes into getRecordFromDocDB method and in the catch block of this method<br><br> def getRecordFromDocDB(reservationKey: String, ratePlanInd: Boolean = false)<br>                        (implicit conf: SparkConf): (Option[ReservationCollationEvent], Boolean) = {<br>    log.info(&quot;Fetching the old reservation record from the documentDB - getRecordFromDocDB()&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    val mongodbClient = getMongoClient(conf, DocumentDBOutput)<br><br>    log.info(&quot; - retrieving the reservation record from documentDB for the reservation key: - &quot; +<br>      reservationKey, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    Try {<br>      val databaseName = conf.get(DocumentDbOutputDatabase)<br>      val collectionName = conf.get(DocumentDbOutputCollection)<br><br>      log.info(&quot; - Reservation Key from documentDB =&gt; &quot; + reservationKey,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      val db = mongodbClient.getDatabase(databaseName)<br>      val collection = db.getCollection(collectionName)<br>      val filter = Document(&quot;_id&quot; -&gt; reservationKey)<br><br>      collection.find(filter).first()<br>    } match {<br>      case Success(oldReservationStr) =&gt;<br>        if (oldReservationStr != null) {<br>          println(&quot;check to come under success&quot;)<br>          val oldReservation = parseOldReservationRecord(oldReservationStr.toJson)<br>          oldReservation match {<br>            case None =&gt; (None, false)<br><br>            case Some(oldReservation) =&gt;<br>              (Some(ReservationCollationEvent(<br>                DefaultEventInfo(<br>                  EventVersion,<br>                  EventSourceValue,<br>                  EventIdentifierValue,<br>                  RecordNumberValue,<br>                  LocalDateTime.now().toString,<br>                  Map[String, String](),<br>                  EmptyString,<br>                  Seq[String](),<br>                  RetryCounterValue,<br>                  DestinationTopicValue,<br>                  Seq[String](),<br>                  Seq[String]()),<br>                ReservationCollationPayload(oldReservation.toString, &quot;json&quot;, Some(oldReservation)))), false)<br>          }<br>        } else {<br>          println(&quot;checking here to come &quot;)<br>          (None, false)<br>        }<br>     // case Failure(exception: NullPointerException) =&gt; {<br>      case Failure(ex : Exception) =&gt; {<br>        log.error(&quot;Failed while retrieving reservation record from documentDB :: Exception :: getRecordFromDocDB(): &quot;<br>          + ex, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//        new CustomException(exception,<br>//          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)).handle()<br>//        (None, true)<br>        throw ex<br>      }<br>    }<br>  }<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>  ========================================================<br><br><br>  but the issue is why it is going to  getRecordFromDocDB method as it first gets into fetchPMSfromRateplan there it gets the exception as it is not able to connect to mongodb then why instead of going in the catch block of fetchPMSfromRateplan method it is going outside the method and goes into  getRecordFromDocDB method ?<br>  def getLatestReservationRecord(reservationKey: Option[String],<br>                                 reservationRecords: Iterator[ReservationCollationEvent])<br>                                //(implicit conf: SparkConf): NewNOldReservationRecordsEntity = {<br>                                (implicit conf: SparkConf,jobConfigurationDetails: JobConfigurationDetails<br>                                 ,connectionConfigDetails: ConnectionConfigDetails): NewNOldReservationRecordsErrorEntity = {<br><br>    val reservationRecordsList = reservationRecords.toList<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br>        val (docDBReservationRecord, recoverableErrFlag) = getRecordFromDocDB(reservationKey.get)<br><br><br><br>  this is the method  which we deliberatlely giving the wrong password  so that it fails and goes to the  catch exception, the exception we are getting is this<br><br>   Caused by: scala.MatchError: Failure(com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=SCRAM-SHA-1, userName=&#39;oneyieldmaster&#39;, source=&#39;admin&#39;, password=&lt;hidden&gt;, mechanismProperties={}}) (of class scala.util.Failure)<br>  at com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getRecordFromDocDB(ReservationCollationHelper.scala:1148)<br>  at com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getRecordFromDocDB$(ReservationCollationHelper.scala:1138)<br>  at com.marriott.mgpapp.spark.reservation.ingestion.jobs.ReservationIngestionJob.getRecordFromDocDB(ReservationIngestionJob.scala:25)<br>  at com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getLatestRe<br><br>It is also printing println(&quot;yaha par bhi kuch laga ke dekh lete hai kahin yaha na aajaye&quot;) which should not be the case a sit should go under catch after it fails the connection but it is not going in the catch block?<br>but after failing it is not going to the catch block  catch {<br><br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb-fetchPMS: &quot; + ex.printStackTrace())<br>        //List.empty[ReservationCollationEvent]<br>        println(&quot;stop_4&quot;)<br>        throw ex<br><br>    }why??<br><br>    ==============================================================================================================================================<br><br><br><br><br><br>    {&quot;@timestamp&quot;:&quot;2024-10-06T15:33:51.004+05:30&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Exception in task 0.0 in stage 3.0 (TID 3)&quot;,&quot;logger_name&quot;:&quot;org.apache.spark.executor.Executor&quot;,&quot;thread_name&quot;:&quot;Executor task launch worker for task 0.0 in stage 3.0 (TID 3)&quot;,&quot;level&quot;:&quot;ERROR&quot;,&quot;level_value&quot;:40000,&quot;stack_trace&quot;:&quot;com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=SCRAM-SHA-1, userName=&#39;oneyieldmaster&#39;, source=&#39;admin&#39;, password=&lt;hidden&gt;, mechanismProperties={}}\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.wrapException(SaslAuthenticator.java:173)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.access$300(SaslAuthenticator.java:40)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:70)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:47)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.doAsSubject(SaslAuthenticator.java:179)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.authenticate(SaslAuthenticator.java:47)\r\n\tat com.mongodb.internal.connection.InternalStreamConnectionInitializer.authenticateAll(InternalStreamConnectionInitializer.java:151)\r\n\tat com.mongodb.internal.connection.InternalStreamConnectionInitializer.initialize(InternalStreamConnectionInitializer.java:64)\r\n\tat com.mongodb.internal.connection.InternalStreamConnection.open(InternalStreamConnection.java:127)\r\n\tat com.mongodb.internal.connection.UsageTrackingInternalConnection.open(UsageTrackingInternalConnection.java:50)\r\n\tat com.mongodb.internal.connection.DefaultConnectionPool$PooledConnection.open(DefaultConnectionPool.java:390)\r\n\tat com.mongodb.internal.connection.DefaultConnectionPool.get(DefaultConnectionPool.java:106)\r\n\tat com.mongodb.internal.connection.DefaultConnectionPool.get(DefaultConnectionPool.java:92)\r\n\tat com.mongodb.internal.connection.DefaultServer.getConnection(DefaultServer.java:85)\r\n\tat com.mongodb.binding.ClusterBinding$ClusterBindingConnectionSource.getConnection(ClusterBinding.java:114)\r\n\tat com.mongodb.client.internal.ClientSessionBinding$SessionBindingConnectionSource.getConnection(ClientSessionBinding.java:111)\r\n\tat com.mongodb.operation.OperationHelper.withConnectionSource(OperationHelper.java:460)\r\n\tat com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:406)\r\n\tat com.mongodb.operation.FindOperation.execute(FindOperation.java:695)\r\n\tat com.mongodb.operation.FindOperation.execute(FindOperation.java:83)\r\n\tat com.mongodb.client.internal.MongoClientDelegate$DelegateOperationExecutor.execute(MongoClientDelegate.java:179)\r\n\tat com.mongodb.client.internal.FindIterableImpl.first(FindIterableImpl.java:198)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.$anonfun$getRecordFromDocDB$1(ReservationCollationHelper.scala:1159)\r\n\tat scala.util.Try$.apply(Try.scala:213)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getRecordFromDocDB(ReservationCollationHelper.scala:1148)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getRecordFromDocDB$(ReservationCollationHelper.scala:1138)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.jobs.ReservationIngestionJob.getRecordFromDocDB(ReservationIngestionJob.scala:25)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getLatestReservationRecord(ReservationCollationHelper.scala:1596)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.getLatestReservationRecord$(ReservationCollationHelper.scala:1582)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.jobs.ReservationIngestionJob.getLatestReservationRecord(ReservationIngestionJob.scala:25)\r\n\tat com.marriott.mgpapp.spark.reservation.ingestion.helper.ReservationCollationHelper.$anonfun$reservationCollation$2(ReservationCollationHelper.scala:115)\r\n\tat org.apache.spark.sql.KeyValueGroupedDataset.$anonfun$mapGroups$1(KeyValueGroupedDataset.scala:193)\r\n\tat org.apache.spark.sql.execution.MapGroupsExec.$anonfun$doExecute$17(objects.scala:424)\r\n\tat scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)\r\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)\r\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:513)\r\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\r\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\r\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\r\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\r\n\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\r\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\r\n\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223)\r\n\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:302)\r\n\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1508)\r\n\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1435)\r\n\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1499)\r\n\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1322)\r\n\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:376)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:327)\r\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:55)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\r\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)\r\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:329)\r\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\r\n\tat org.apache.spark.scheduler.Task.run(Task.scala:136)\r\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)\r\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)\r\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)\r\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\r\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: com.mongodb.MongoCommandException: Command failed with error 18: &#39;Authentication failed.&#39; on server rms-oy-test-documentdb.cluster-clofdwmpxqfq.us-east-1.docdb.amazonaws.com:27017. The full response is {\&quot;ok\&quot;: 0, \&quot;errmsg\&quot;: \&quot;Authentication failed.\&quot;, \&quot;code\&quot;: 18, \&quot;operationTime\&quot;: {\&quot;$timestamp\&quot;: {\&quot;t\&quot;: 1728209006, \&quot;i\&quot;: 1}}}\r\n\tat com.mongodb.internal.connection.ProtocolHelper.getCommandFailureException(ProtocolHelper.java:179)\r\n\tat com.mongodb.internal.connection.InternalStreamConnection.receiveCommandMessageResponse(InternalStreamConnection.java:293)\r\n\tat com.mongodb.internal.connection.InternalStreamConnection.sendAndReceive(InternalStreamConnection.java:255)\r\n\tat com.mongodb.internal.connection.CommandHelper.sendAndReceive(CommandHelper.java:83)\r\n\tat com.mongodb.internal.connection.CommandHelper.executeCommand(CommandHelper.java:33)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.sendSaslContinue(SaslAuthenticator.java:134)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator.access$200(SaslAuthenticator.java:40)\r\n\tat com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:67)\r\n\t... 73 common frames omitted\r\n&quot;} this is the error i got , why after getting into this method it goes into getRecordFromDocDB method and in the catch block of this method afte that is fails why it does not get into the catch class of  override def process(args: Map[String, String]): Unit = {<br>  try{<br>    val (newNOldRecordsDS) = reservationCollation(filteredReservationDS, sparkSession,errorAccumulator)<br>      persistDS(newNOldRecordsDS,<br>        payloadEvent,<br>        connectionConfigDetails.brokerconfig.get,<br>        jobConfigDetails.checkpointdirectory.get,<br>        jobConfigDetails.jobname,<br>        s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval)<br>    }<br>    catch {<br><br>      case exception: Exception =&gt;<br>        println(&quot;is it entering here-------------------------&quot;)<br>        val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;recoverable checkpoint&quot;)<br>        produceRecoverableData(finalDF)   }<br>    sparkSession.streams.awaitAnyTermination<br>  }<br><br><br><br>answer==============================================================<br><br>Why the Catch Block in process Method is Not Triggered<br>The reason your catch block in the process method is not being triggered could be related to how exceptions are propagated in the code. Here are a few possibilities:<br><br>Exception Handling in getRecordFromDocDB: If getRecordFromDocDB or any method it calls is handling the exception internally, the exception will not propagate up to the process method. Check if there are any try-catch blocks in getRecordFromDocDB or its dependencies that might be catching the exception and handling it without rethrowing.<br><br><br>Threading and Asynchronous Execution: If the code is running on a separate thread or is part of an asynchronous operation, the exception might not propagate to the main thread where process is being executed. Ensure that any asynchronous operations correctly propagate exceptions back to the main thread.<br><br><br>Spark&#39;s Task Execution: In Spark, tasks run on separate executors, and exceptions in these tasks might not directly affect the control flow of the driver program. Instead, they might be captured in the task&#39;s error logs. Make sure the exception in getRecordFromDocDB is not being logged and swallowed by Spark&#39;s task handling.<br><br><br>Missing Exception Rethrow: If the getRecordFromDocDB method has a catch block but does not rethrow the exception, it will not propagate up to process. Check if there&#39;s a catch block in getRecordFromDocDB that swallows the exception without rethrowing it.<br><br><br>Steps to Diagnose and Fix<br><br><br>Check Internal Exception Handling: Review the getRecordFromDocDB method and any methods it calls to see if exceptions are being caught and not rethrown.<br><br><br>Review Spark Job Execution: Ensure that the Spark job&#39;s error handling is configured to propagate exceptions. Check the driver logs for any unhandled exceptions.<br><br><br>Log Additional Details: Add logging inside the getRecordFromDocDB method and other critical points to track where the exception occurs and where it might be getting caught.<br><br><br>Thread and Async Operations: If any operations are asynchronous, ensure they correctly handle and propagate exceptions.<br>Once you identify where the exception is being handled or swallowed, you can adjust the code to ensure it correctly propagates up to the process method&#39;s catch block.<br><br><br><br><br><br>====================================================================================================================================================<br><br><br><br><br><br><br>correct the syntax issue in this line   val data = consumeDataFromEEH(InputType.Avro)<br> no it is some bracket issue it was working fine before i put def in try catch correct that syntax issue as it is giving wrong forward reference in here<br> <br> override def process(args: Map[String, String]): Unit = {<br><br><br><br>    implicit val implicitCouchDetails: ImplicitCouchDetails = ImplicitCouchDetails(&quot;RMS-RES-JOURNAL&quot;, &quot;SELECT META().id as `documentId`, `bucketName`.* FROM `bucketName` WHERE queryRatePlanList&quot;)<br><br><br>    /**<br>     * Consuming EEH Kafka topic data<br>     * Use val file = &quot;HotelReservationV8.avsc&quot; when From_avro Function is used<br>     * Using UDF method, File is not required<br>     * */<br>    import sparkSession.implicits._<br>    val data = consumeDataFromEEH(InputType.Avro)<br><br>    log.info(&quot;Read Data From EEH&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    /**<br>     * Method call to Filtration Step<br>     */<br>    val (filteredReservationDS, irrecoverableDF, recoverableS3DF) = reservationFiltration(data)<br>    val payloadEvent = filteredReservationDS.map(x =&gt; x.reservationPayload.payLoad).toDF<br>    //println(&quot;5 Aug - Data after filtration step - &quot;)<br>    //filteredReservationDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Reservation Filtration Success &quot;, customMetadata)<br><br>    log.info(&quot;Initiating Irrecoverable Errors messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    try {<br>      val errorAccumulator: LongAccumulator = sparkSession.sparkContext.longAccumulator(&quot;Error Accumulator&quot;)<br><br>      val (newNOldRecordsDS) = reservationCollation(filteredReservationDS, sparkSession, errorAccumulator)<br><br>      log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      try {<br>        /**<br>         * Method call for event step<br>         */<br>        persistDS(newNOldRecordsDS,<br>          payloadEvent,<br>          connectionConfigDetails.brokerconfig.get,<br>          jobConfigDetails.checkpointdirectory.get,<br>          jobConfigDetails.jobname,<br>          s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Error while writing to couchbase :: Exception :: reservationCollation():&quot; + exception.printStackTrace(),<br>            CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>          new DocumentDBException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      }<br><br><br>      sparkSession.streams.awaitAnyTermination<br><br>    } catch {<br>      case ex: Exception =&gt;<br>        // This catch block will be executed if any exception is thrown from mapGroups<br>        log.error(&quot;Exception caught in main process&quot;, ex)<br>        //        if (errorAccumulator.value &gt; 0) {<br>        //          log.info(&quot;Accumulator is greater than 0&quot;)<br>        val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;recoverable checkpoint&quot;)<br>        produceRecoverableData(finalDF)<br>      // }<br>    }<br><br>    // Method definitions...<br><br>    def persistDS(newNOldRecordsDS: Dataset[ReservationEventsEntity],<br>                  payloadEvent: DataFrame,<br>                  kafkaConfig: MessageBrokerDetails,<br>                  checkpointDir: Map[String, String],<br>                  jobName: String,<br>                  s3bucket: String,<br>                  kafkaMaxSize: Float,<br>                  triggerInterval: Int,<br>                  //                sparkSession: SparkSession,<br>                  //                connectionConfigDetails: ConnectionConfigDetails<br>                 )<br>                 (implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails<br>                  , sparkSession: SparkSession, connectionConfigDetails: ConnectionConfigDetails<br>                 ): Unit = {<br><br>      import sparkSession.implicits._<br>      newNOldRecordsDS<br>        .writeStream<br>        .foreachBatch((batchDS: org.apache.spark.sql.Dataset[com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationEventsEntity], _: Long) =&gt; {<br><br>          batchDS.persist()<br><br>          batchDS.foreach { row =&gt;<br>            val documentDBSinkObject = new DocumentDBKafkaSink(conf, kafkaConfig, kafkaMaxSize)<br>            documentDBSinkObject.open(1.toLong, 2)<br>            documentDBSinkObject.process(row)<br>          }<br><br><br><br>          val resEventOutDS = reservationEvent(batchDS, payloadEvent)<br>          //val resEventOutDF = resEventOutDS.toDF()<br>          val checkPointPathKafka: Option[String] = Option(S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationEventCheckpointDir)))<br><br>          logger.info(&quot;checkpoint path:&quot; + checkPointPathKafka)<br><br>          val resEventOutFinalDF = resEventOutDS.select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>          produceDataToEEH(resEventOutFinalDF)<br>        })<br>        .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationCollationCheckpointDir)))<br>        .trigger(Trigger.ProcessingTime(triggerInterval))<br>        .start()<br><br>    }<br><br>    def produceDataToEEH(data: DataFrame)(implicit checkpointPath: Option[String]): Unit = {<br><br>      val input = InputType.Avro<br>      val checkPointPathKafka: Option[String] = checkpointPath<br>      val destinationTopic = KafkaProperties.destinationTopic<br>      val schemaPath = &quot;reservationEventSchema.avsc&quot;<br>      try {<br><br>        val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Batch)<br>          .withFilePath(Some(schemaPath))<br>          .withCaseClass(Some(classOf[EEHData]))<br>          .withCheckPoint(checkPointPathKafka)<br>          .withInputType(InputType.Avro)<br>          .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>          .build()<br><br>        producerHelper.produce()<br>        log.info(&quot;pushed data into Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while pushing data into kafka topic :: Exception:&quot; + exception.printStackTrace(),<br>            customMetadata)<br>          new KafkaException(exception, customMetadata).handle()<br>      }<br>    }<br><br>    def consumeDataFromEEH(inputType: InputType.Type)(implicit spark: SparkSession): DataFrame = {<br><br>      val sourceTopic = KafkaProperties.sourceTopic<br>      val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:-1, &quot;1&quot;:-1, &quot;2&quot;:-1, &quot;3&quot;:-1, &quot;4&quot;: 57064, &quot;5&quot;: -1, &quot;6&quot;: -1, &quot;7&quot;: -1}}&quot;&quot;&quot;<br>      //  val startingOffset: String = &quot;latest&quot;<br>      val input = inputType<br><br>      try {<br>        val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>          .withStartingOffsets(startingOffset)<br>          .withInputType(input)<br>          .withDeserializer(Deserializer.deserializerWithUdf)<br>          .build()<br><br>        readStreamData.consume().select(&quot;value&quot;)<br><br><br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while fetch data fram kafka topic&quot;, customMetadata)<br>          new KafkaException(exception, customMetadata).handle()<br>          val emptyDataFrame: DataFrame = null<br>          emptyDataFrame<br>      }<br>    }<br><br><br>  }<br>}<br><br><br>======================================================================================================<br>fix this for the new try catch <br><br>class ReservationIngestionJob(implicit sparkSession: SparkSession,<br>                              jobConfigDetails: JobConfigurationDetails,<br>                              connectionConfigDetails: ConnectionConfigDetails)<br>  extends ReservationIngestionHelper with ReservationCollationHelper<br>    with ReservationFiltrationHelper with ReservationEventHelper<br>    with StreamingJob with Serializable {<br><br>  this: ReservationIngestionHelper =&gt;<br>  override val jobName: String = jobConfigDetails.jobname<br>  override val jobIdentifier: String = jobConfigDetails.jobidentifier<br>  override val triggerInterval: Int = jobConfigDetails.triggerinterval.get<br>  //TO-DO Remove Couchbase Details<br>  /*<br>  val couchDbDetails = connectionConfigDetails.couchbaseconfig.get<br>  implicit val conf: SparkConf = getCouchSparkConf(couchDbDetails, couchBucketName)*/<br>  override val checkPointDirectory: Map[String, String] = jobConfigDetails.checkpointdirectory.get<br><br>  //implicit val couchBucketName: String = jobConfigDetails.couchbasemap.get(ReservationCollationBucket)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val customMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)<br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br>  val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br>  var errorDs: Dataset[ReservationCollationEvent] = _<br><br>  override def process(args: Map[String, String]): Unit = {<br><br><br>    /*    val bucketName: String = jobConfigDetails.couchbasemap.get(HPPRatesBucket)<br>        val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)*/<br><br>    implicit val implicitCouchDetails: ImplicitCouchDetails = ImplicitCouchDetails(&quot;RMS-RES-JOURNAL&quot;, &quot;SELECT META().id as `documentId`, `bucketName`.* FROM `bucketName` WHERE queryRatePlanList&quot;)<br><br>    //    val checkpointDirMap = jobConfigDetails.checkpointdirectory.get<br>    //    val irrecoverableErrorCheckpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(IrrecoverableErrorCheckpointDir).get))<br>    //    val recoverableErrorS3Checkpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(RecoverableErrorS3CheckpointDir).get))<br>    //    val recoverableErrorDocDBCheckpoint: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkPointDirectory.get(RecoverableErrorDocDBCheckpointDir).get))<br>    //    val trigger: Trigger = Trigger.ProcessingTime(triggerInterval)<br><br>    /**<br>     * Consuming EEH Kafka topic data<br>     * Use val file = &quot;HotelReservationV8.avsc&quot; when From_avro Function is used<br>     * Using UDF method, File is not required<br>     * */<br>    import sparkSession.implicits._<br>    val data = consumeDataFromEEH(InputType.Avro)<br><br>    log.info(&quot;Read Data From EEH&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    /**<br>     * Method call to Filtration Step<br>     */<br>    val (filteredReservationDS, irrecoverableDF, recoverableS3DF) = reservationFiltration(data)<br>    val payloadEvent = filteredReservationDS.map(x =&gt; x.reservationPayload.payLoad).toDF<br>    //println(&quot;5 Aug - Data after filtration step - &quot;)<br>    //filteredReservationDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Reservation Filtration Success &quot;, customMetadata)<br><br>    log.info(&quot;Initiating Irrecoverable Errors messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>  <br>    val errorAccumulator: LongAccumulator = sparkSession.sparkContext.longAccumulator(&quot;Error Accumulator&quot;)<br><br>    /**<br>     * Method call to Collation Step<br>     */<br>    val (newNOldRecordsDS, recoverableDocDBDF) = reservationCollation(filteredReservationDS, sparkSession,errorAccumulator)<br>    //println(&quot;5 Aug - Data after collation step - &quot;)<br>    //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>    log.info(&quot;Initiating Recoverable Errors of Document DB messages push to Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>//    val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>//    println(&quot;printing schema of  recoverableEvent&quot;)<br>//    recoverableEvent.printSchema()<br>    println(&quot;errorAccumulator value is &quot; , errorAccumulator )<br>    //if (errorAccumulator.value &gt; 0) {<br>      log.info(&quot;accumulator is greater than 0&quot;)<br>      val finalDF = payloadEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      println(&quot;recoverable checkpoint&quot;)<br>      produceRecoverableData(finalDF)<br>   // }<br><br>    try {<br>      /**<br>       * Method call for event step<br>       */<br>      persistDS(newNOldRecordsDS,<br>        payloadEvent,<br>        connectionConfigDetails.brokerconfig.get,<br>        jobConfigDetails.checkpointdirectory.get,<br>        jobConfigDetails.jobname,<br>        s3bucket, jobConfigDetails.kafkamaxrequestsize.get, triggerInterval)<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Error while writing to couchbase :: Exception :: reservationCollation():&quot; + exception.printStackTrace(),<br>          CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>        new DocumentDBException(exception, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>    }<br>    sparkSession.streams.awaitAnyTermination<br>  }<br><br>  def persistDS(newNOldRecordsDS: Dataset[ReservationEventsEntity],<br>                payloadEvent:DataFrame,<br>                kafkaConfig: MessageBrokerDetails,<br>                checkpointDir: Map[String, String],<br>                jobName: String,<br>                s3bucket: String,<br>                kafkaMaxSize: Float,<br>                triggerInterval: Int,<br>//                sparkSession: SparkSession,<br>//                connectionConfigDetails: ConnectionConfigDetails<br>               )<br>               (implicit conf: SparkConf, implicitCouchDetails: ImplicitCouchDetails<br>                , sparkSession: SparkSession, connectionConfigDetails: ConnectionConfigDetails<br>               ): Unit = {<br><br>    import sparkSession.implicits._<br>    newNOldRecordsDS<br>      .writeStream<br>      .foreachBatch((batchDS: org.apache.spark.sql.Dataset[com.marriott.mgpapp.spark.reservation.ingestion.entity.ReservationEventsEntity], _: Long) =&gt; {<br><br>        batchDS.persist()<br><br>        batchDS.foreach { row =&gt;<br>          val documentDBSinkObject = new DocumentDBKafkaSink(conf, kafkaConfig, kafkaMaxSize)<br>          documentDBSinkObject.open(1.toLong, 2)<br>          documentDBSinkObject.process(row)<br>        }<br>        val resEventOutDS = reservationEvent(batchDS,payloadEvent)<br>        //val resEventOutDF = resEventOutDS.toDF()<br>        val checkPointPathKafka: Option[String] = Option(S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationEventCheckpointDir)))<br><br>        logger.info(&quot;checkpoint path:&quot; + checkPointPathKafka)<br><br>        val resEventOutFinalDF = resEventOutDS.select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>        produceDataToEEH(resEventOutFinalDF)<br>      })<br>      .option(&quot;checkpointLocation&quot;, S3Util.getCheckPointDirectory(s3bucket, jobName, checkpointDir(ReservationCollationCheckpointDir)))<br>      .trigger(Trigger.ProcessingTime(triggerInterval))<br>      .start()<br><br>  }<br><br>  def produceDataToEEH(data: DataFrame)(implicit checkpointPath: Option[String]): Unit = {<br><br>    val input = InputType.Avro<br>    val checkPointPathKafka: Option[String] = checkpointPath<br>    val destinationTopic = KafkaProperties.destinationTopic<br>    val schemaPath = &quot;reservationEventSchema.avsc&quot;<br>    try {<br><br>      val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(schemaPath))<br>        .withCaseClass(Some(classOf[EEHData]))<br>        .withCheckPoint(checkPointPathKafka)<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>      log.info(&quot;pushed data into Kafka&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while pushing data into kafka topic :: Exception:&quot; + exception.printStackTrace(),<br>          customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>    }<br>  }<br><br>  def consumeDataFromEEH(inputType: InputType.Type)(implicit spark: SparkSession): DataFrame = {<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    val startingOffset = &quot;&quot;&quot;{&quot;commerce.reservation.reservation.sync&quot;:{&quot;0&quot;:-1, &quot;1&quot;:-1, &quot;2&quot;:-1, &quot;3&quot;:-1, &quot;4&quot;: 52780, &quot;5&quot;: -1, &quot;6&quot;: -1, &quot;7&quot;: -1}}&quot;&quot;&quot;<br>  //  val startingOffset: String = &quot;latest&quot;<br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br><br>      readStreamData.consume().select(&quot;value&quot;)<br><br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data fram kafka topic&quot;, customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>        val emptyDataFrame: DataFrame = null<br>        emptyDataFrame<br>    }<br>  }<br><br><br>}<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>==================================================================================<br><br>iti is pritning the message {&quot;@timestamp&quot;:&quot;2024-10-06T20:56:49.196+05:30&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;pushed data into Kafka2&quot;,&quot;logger_name&quot;:&quot;com.marriott.mgpapp.spark.reservation.ingestion.jobs.ReservationIngestionJob&quot;,&quot;thread_name&quot;:&quot;main&quot;,&quot;level&quot;:&quot;INFO&quot;,&quot;level_value&quot;:20000,&quot;app&quot;:&quot;Reservation&quot;,&quot;trace-id&quot;:&quot;traceId&quot;,&quot;project&quot;:<br><br>but i cannot see any data in the kafka ....why?<br><br><br>def produceRecoverableData(data: DataFrame)(implicit connectionConfigDetails: ConnectionConfigDetails,jobConfigurationDetails: JobConfigurationDetails) = {<br>   log.info(&quot;Entered in the Recoverable method&quot;)<br>    data.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>    val input = InputType.Avro<br>    val customMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData)<br>    val s3bucket = connectionConfigDetails.bucketconfig.get(defaultBucket)<br>    val checkpointDirectory = jobConfigurationDetails.checkpointdirectory.get(&quot;ReservationIngestionCheckpoint&quot;)<br>    val checkPointPathKafka: String =  S3Util.getCheckPointDirectory(<br>      s3bucket,<br>      jobName,<br>      checkpointDirectory<br>    )<br>    val destinationTopic = KafkaProperties.recoverableErrorTopic<br>    val schemaPath = &quot;recoverable-DLQ.avsc&quot;<br>    println(&quot;yaha tak aaja ek baar&quot;)<br>    try {<br>      val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Streaming)<br>        .withFilePath(Some(schemaPath))<br>        .withCaseClass(Some(classOf[HotelReservation]))<br>        .withInputType(InputType.Avro)<br>        .withCheckPointPath(Some(checkPointPathKafka))<br>        .withIsSingleStreaming(false)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>      log.info(&quot;pushed data into Kafka2&quot;, CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while pushing data into kafka topic 2 for recoverable :: Exception:&quot; + exception.printStackTrace(),<br>          customMetadata)<br>        new KafkaException(exception, customMetadata).handle()<br>    }<br>  }<br>}</pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=5eadb665d1e0" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[My Sap Backup -1.]]></title>
            <link>https://medium.com/@ankitjuneja/my-sap-backup-1-38eb6c0aa0a4?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/38eb6c0aa0a4</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Fri, 22 Aug 2025 02:52:25 GMT</pubDate>
            <atom:updated>2025-08-22T04:58:47.047Z</atom:updated>
            <content:encoded><![CDATA[<p>iheart</p><pre><br><br>dim_profile_phone_df.select(&quot;profile_id&quot;, &quot;name&quot;, &quot;phone_number&quot;).filter(col(&quot;profile_id&quot;)== 9950577377).show(TRUNCATE=False)<br><br><br>id_res_union_df.filter(col(&quot;source_id&quot;)== &quot;9950577377&quot;).show()<br><br><br>filter_matched_df.filter(col(&quot;source_id&quot;)== &quot;9950577377&quot;).show()<br><br><br>display(id_res_send_df)<br><br>expected_data_df   =expected_data_df.drop(&quot;etl_date&quot;)<br><br><br>#1. **Linking New Records to Existing TU_ID**:<br># AP-189: Also Exclude IDs (IPs, MAIDs) from &#39;0&#39; and negative profile_ids in TU<br>df_update = spark.sql(f&quot;&quot;&quot;<br>    WITH matched_dataset AS (<br>        SELECT * <br>        FROM silver.id_resolution_send_progress send_progress<br>        JOIN landing.transunion_id_resolution_response response<br>            ON cast(send_progress.id_resolution_attempt_id as string) = response.CustomerID<br>        WHERE response.Extern_TUID IS NOT NULL and send_progress.source_id &gt; &#39;0&#39;<br>  <br>  <br>  <br>  <br>  <br>  <br>  prod.triton_individual<br>  <br>  # Joining the send progress and response tables to match the dataset.<br># Also AP-189:Exclude IDs (IPs, MAIDs) from &#39;0&#39; and negative profile_ids in TU<br>matched_dataset_df = send_progress_df.join(<br>    response_df,<br>    col(&quot;id_resolution_attempt_id&quot;).cast(&quot;string&quot;) == col(&quot;CustomerID&quot;),<br>    &quot;inner&quot;<br>).filter((col(&quot;Extern_TUID&quot;).isNotNull()) &amp; (col(&quot;source_id&quot;) &gt; &#39;0&#39;))<br><br><br><br><br># Filter out only existing records from the SQL query which is joining two tables &#39;send_progress&#39; and &#39;response&#39;. Also, select distinct records of new entries where a TUID is not already linked. <br># AP-189: Also Exclude IDs (IPs, MAIDs) from &#39;0&#39; and negative profile_ids in TU<br>df_existed_records = spark.sql(f&quot;&quot;&quot;<br>    WITH matched_dataset AS (<br>        SELECT send_progress.*, response.*<br>        FROM silver.id_resolution_send_progress send_progress<br>        JOIN landing.transunion_id_resolution_response response<br>            ON cast(send_progress.id_resolution_attempt_id as string) = response.CustomerID<br>        WHERE response.Extern_TUID IS NOT NULL and send_progress.source_id &gt; &#39;0&#39; <br>    ) </pre><p>Lambda:</p><pre>import os<br>import boto3<br>import logging<br>from pathlib import Path<br>from gpg_utils import gpg_import_keys, gpg_encrypt<br>import json<br><br># Configure logging <br>logger = logging.getLogger() <br>logger.setLevel(logging.INFO)<br><br>s3 = boto3.client(&#39;s3&#39;)<br>secrets_manager = boto3.client(&#39;secretsmanager&#39;)<br><br>def fetch_gpg_secrets(secret_name):<br>    response = secrets_manager.get_secret_value(SecretId=secret_name)<br>    secret = json.loads(response[&#39;SecretString&#39;])<br>    return secret[&#39;public_key&#39;], secret[&#39;private_key&#39;], secret[&#39;recipient_key&#39;], secret[&#39;passphrase&#39;]<br>    <br>def lambda_handler(event, context):<br>    # Fetch GPG secrets from Secrets Manager<br>    secret_name = &#39;ihm_encryption_decryption_keys&#39;  # Replace with your secret name<br>    public_key, private_key, recipient_key, passphrase = fetch_gpg_secrets(secret_name)<br>    <br>    logger.info(&#39;GPG keys and passphrase fetched from Secrets Manager&#39;)<br><br>    # S3 bucket and folders (from environment variables) <br>    s3_bucket = os.getenv(&#39;S3_BUCKET&#39;) <br>    archive_prefix = os.getenv(&#39;ARCHIVE_PREFIX&#39;) <br>    to_transunion_bucket = os.getenv(&#39;TO_TRANSUNION_BUCKET&#39;) <br>    to_transunion_prefix = os.getenv(&#39;TO_TRANSUNION_PREFIX&#39;)<br><br>    # There should be only one record for the done.psv file<br>    s3_key = event[&#39;Records&#39;][0][&#39;s3&#39;][&#39;object&#39;][&#39;key&#39;]<br>    logger.info(&#39;Processing file: %s from bucket: %s&#39;, s3_key, s3_bucket)<br><br>    # Ensure this is the done.psv file<br>    if not s3_key.endswith(&#39;_Part_Done.psv&#39;):<br>        raise Exception(&#39;Triggered by a non-done.psv file, which is not expected.&#39;)<br><br>    # Determine the prefix for listing all files in the folder<br>    prefix = os.path.dirname(s3_key) + &quot;/&quot;<br>    logger.info(&#39;Listing files in folder: %s&#39;, prefix)<br><br>    # List all files in the prefix folder<br>    response = s3.list_objects_v2(Bucket=s3_bucket, Prefix=prefix)<br>    for obj in response.get(&#39;Contents&#39;, []):<br>        file_key = obj[&#39;Key&#39;]<br>        file_name = os.path.basename(file_key)<br>        file_to_encrypt_path = f&#39;/tmp/{file_name}&#39;<br>        encrypted_output_path = f&#39;/tmp/{file_name}.gpg&#39;<br>        archive_key = f&quot;{archive_prefix}{file_name}&quot;<br>        encrypted_key = f&quot;{to_transunion_prefix}{file_name}&quot;<br><br>        # Skip directories<br>        if file_key.endswith(&#39;/&#39;):<br>            continue<br><br>        # Check if the file ends with &quot;_Part_Done.psv&quot;<br>        if file_key.endswith(&#39;_Part_Done.psv&#39;):<br>            try:<br>                # Move the &quot;done.psv&quot; file to the archive folder and to_transunion folder without encrypting<br>                copy_source = {&#39;Bucket&#39;: s3_bucket, &#39;Key&#39;: file_key}<br>                s3.copy_object(Bucket=s3_bucket, CopySource=copy_source, Key=archive_key, Metadata={&#39;archived&#39;: &#39;true&#39;}, MetadataDirective=&#39;REPLACE&#39;)<br>                s3.copy_object(Bucket=to_transunion_bucket, CopySource=copy_source, Key=encrypted_key)<br>                s3.delete_object(Bucket=s3_bucket, Key=file_key)<br>                logger.info(&#39;_Part_Done.psv file moved to archive and to_transunion: %s&#39;, archive_key)<br>            except Exception as e:<br>                logger.error(f&quot;Error moving _Part_Done.psv file to archive and to_transunion: {e}&quot;)<br>                raise Exception(f&quot;Failed to move {file_key} to archive and to_transunion in bucket {s3_bucket}.&quot;)<br>            continue<br><br>        # Download the file to encrypt from S3<br>        try:<br>            s3.download_file(s3_bucket, file_key, file_to_encrypt_path)<br>            logger.info(&#39;File downloaded successfully: %s&#39;, file_to_encrypt_path)<br>        except Exception as e:<br>            logger.error(f&quot;Error downloading file from S3: {e}&quot;)<br>            raise Exception(f&quot;Failed to download {file_key} from bucket {s3_bucket}. Ensure the file exists and the path is correct.&quot;)<br><br>        # Import the public key<br>        code, stdout, stderr = gpg_import_keys(public_key)<br>        if code &gt; 0:<br>            logger.error(f&#39;gpg_import_keys failed with code {code}: {stdout} {stderr}&#39;)<br>            raise Exception(f&#39;gpg_import_keys failed with code {code}: {stdout} {stderr}&#39;)<br><br>        logger.info(&#39;Public key import successful&#39;)<br><br>        # Perform encryption<br>        logger.info(&#39;Performing Encryption of %s&#39;, file_to_encrypt_path)<br>        code, stdout, stderr = gpg_encrypt(file_to_encrypt_path, encrypted_output_path, [recipient_key])<br>        if code &gt; 0:<br>            logger.error(f&#39;gpg_encrypt failed with code {code}: {stderr}&#39;)<br>            raise Exception(f&#39;gpg_encrypt failed with code {code}: {stderr}&#39;)<br><br>        logger.info(&#39;Encryption successful: %s&#39;, encrypted_output_path)<br><br>        # Upload encrypted file to the specified S3 bucket<br>        try:<br>            s3.upload_file(encrypted_output_path, to_transunion_bucket, encrypted_key)<br>            logger.info(&#39;Encrypted file uploaded to S3: %s&#39;, encrypted_key)<br>        except Exception as e:<br>            logger.error(f&quot;Error uploading file to S3: {e}&quot;)<br>            raise Exception(f&quot;Failed to upload {encrypted_output_path} to bucket {to_transunion_bucket}.&quot;)<br><br>        # Move the original file to the archive folder<br>        try:<br>            copy_source = {&#39;Bucket&#39;: s3_bucket, &#39;Key&#39;: file_key}<br>            s3.copy_object(Bucket=s3_bucket, CopySource=copy_source, Key=archive_key, Metadata={&#39;archived&#39;: &#39;true&#39;}, MetadataDirective=&#39;REPLACE&#39;)<br>            s3.delete_object(Bucket=s3_bucket, Key=file_key)<br>            logger.info(&#39;Original file archived in S3: %s&#39;, archive_key)<br>        except Exception as e:<br>            logger.error(f&quot;Error archiving the file in S3: {e}&quot;)<br>            raise Exception(f&quot;Failed to archive {file_key} in bucket {s3_bucket}.&quot;)<br><br>    return {<br>        &#39;statusCode&#39;: 200,<br>        &#39;body&#39;: &#39;Encryption and archiving successful for all files!&#39;<br>    }</pre><p>table create:</p><pre>-- Databricks notebook source<br>-- Create all Audience Platform tables for silver zone schema<br><br>USE SCHEMA silver;<br><br>-- Create the dim_profile table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_profile (<br>    dt date NOT NULL,<br>    profile_id BIGINT NOT NULL,  <br>    mongo_user_id STRING,<br>    account_type_id INT NOT NULL,<br>    email_address STRING NOT NULL,<br>    is_email_valid BOOLEAN NOT NULL,<br>    name STRING,<br>    phone_number STRING,<br>    is_phone_valid BOOLEAN NOT NULL,<br>    terminal_id INT NOT NULL,<br>    birth_year INT,<br>    gender_cd STRING NOT NULL,<br>    zip_code STRING,<br>    account_creation_date TIMESTAMP,<br>    registration_date TIMESTAMP NOT NULL,<br>    last_modified_date DATE NOT NULL,<br>    is_email BOOLEAN NOT NULL,<br>    is_verified_email BOOLEAN NOT NULL,<br>    is_anonymous BOOLEAN NOT NULL,<br>    share_profile BOOLEAN NOT NULL,<br>    third_party_optout BOOLEAN NOT NULL,<br>    etl_date TIMESTAMP NOT NULL<br>)<br>USING DELTA<br>CLUSTER BY (profile_id) ;<br><br><br>-- COMMAND ----------<br><br><br>-- Create the fct_live_radio table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS fct_live_radio (<br>   dt date NOT NULL,<br>   profile_id BIGINT NOT NULL, <br>   user_id STRING, <br>   listener_id STRING, <br>   ip_address STRING NOT NULL, <br>   is_ip_address_valid BOOLEAN NOT NULL, <br>   user_agent STRING, <br>   mobile_advertising_id STRING, <br>   is_maid_valid BOOLEAN NOT NULL,<br>   terminal_id INT NOT NULL, <br>   client_type_id INT NOT NULL,<br>   device_name_id INT NOT NULL, <br>   carrier_id INT,<br>   station_call_letters_id INT, <br>   stream_id INT,<br>   player_type_id INT NOT NULL, <br>   played_from_id INT,<br>   zip_code STRING, <br>   session_start_date TIMESTAMP NOT NULL,  <br>   session_end_date TIMESTAMP NOT NULL, <br>   is_mobile BOOLEAN NOT NULL,<br>   is_fb_broadcast BOOLEAN NOT NULL, <br>   etl_date TIMESTAMP NOT NULL<br>)<br>USING DELTA<br>PARTITIONED BY (dt) ;<br><br><br><br><br>-- COMMAND ----------<br><br>-- -- Create the fct_custom_talk table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS fct_custom_talk(<br>    dt DATE NOT NULL,  <br>    profile_id BIGINT NOT NULL,         <br>    session_id STRING NOT NULL,         <br>    player_key STRING ,         <br>    ip_address STRING NOT NULL,         <br>    is_ip_address_valid BOOLEAN NOT NULL,<br>    mobile_advertising_id STRING,       <br>    is_maid_valid BOOLEAN NOT NULL,              <br>    subscription_type_cd STRING NOT NULL,                 <br>    device_name_id INT NOT NULL,        <br>    station_id STRING NOT NULL,         <br>    station_type_cd STRING NOT NULL,    <br>    host_uri_id BIGINT NOT NULL,        <br>    played_from_id INT NOT NULL,        <br>    track_id INT ,              <br>    complete_stream_reason_id INT,      <br>    reported_date TIMESTAMP NOT NULL,   <br>    elapsed_time INT NOT NULL,          <br>    stream_completed_count INT NOT NULL,<br>    stream_init_count INT NOT NULL,     <br>    stream_started_count INT NOT NULL,  <br>    stream_zero_count INT NOT NULL,     <br>    etl_date TIMESTAMP NOT NULL         <br>)<br>USING delta<br>-- Partition the table by the &#39;dt&#39; column<br>PARTITIONED BY (dt);<br><br>-- COMMAND ----------<br><br>-- Create the fct_custom_radio table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS fct_custom_radio (<br>   dt date NOT NULL,<br>   profile_id BIGINT NOT NULL,<br>   session_id STRING NOT NULL,<br>   player_key STRING ,<br>   ip_address STRING NOT NULL,<br>   is_ip_address_valid BOOLEAN NOT NULL,<br>   mobile_advertising_id STRING ,<br>   is_maid_valid BOOLEAN NOT NULL,<br>   subscription_type_cd STRING NOT NULL,<br>   device_name_id INT NOT NULL,<br>   station_id STRING ,<br>   station_type_cd STRING NOT NULL,<br>   host_uri_id BIGINT NOT NULL ,<br>   played_from_id INT NOT NULL,<br>   content_type_cd STRING NOT NULL,<br>   artist_id INT ,<br>   track_id INT ,<br>   complete_stream_reason_cd INT ,<br>   reported_date TIMESTAMP NOT NULL,<br>   elapsed_time INT NOT NULL,<br>   stream_completed_count INT NOT NULL,<br>   stream_init_count INT NOT NULL,<br>   stream_started_count INT NOT NULL,<br>   stream_zero_count INT NOT NULL,<br>   etl_date TIMESTAMP NOT NULL<br>)<br>USING DELTA<br>PARTITIONED BY (dt);<br><br>-- COMMAND ----------<br><br>-- Create the id_resolution_send_progress table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS id_resolution_send_progress (<br>    id_resolution_attempt_id BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1) PRIMARY KEY,<br>    id_resolution_partner_cd STRING NOT NULL,<br>    source_type_cd STRING NOT NULL,<br>    source_id STRING NOT NULL,<br>    email_address STRING,<br>    phone_number STRING,<br>    ip_address STRING,<br>    user_agent STRING,<br>    mobile_advertising_id STRING,<br>    id_resolution_status_cd STRING NOT NULL,<br>    id_resolution_send_date TIMESTAMP NOT NULL,<br>    etl_date TIMESTAMP NOT NULL<br>)<br>USING delta<br>CLUSTER BY (id_resolution_attempt_id);<br><br>-- COMMAND ----------<br><br>-- Create the id_resolution_send_history table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS id_resolution_send_history (<br>    id_resolution_attempt_id INT,<br>    id_resolution_partner_cd STRING NOT NULL,<br>    source_type_cd STRING NOT NULL,<br>    source_id STRING NOT NULL,<br>    email_address STRING,<br>    phone_number STRING,<br>    ip_address STRING,<br>    user_agent STRING,<br>    mobile_advertising_id STRING,<br>    id_resolution_send_date TIMESTAMP NOT NULL,<br>    etl_date TIMESTAMP NOT NULL<br>)<br>USING delta<br>CLUSTER BY (id_resolution_attempt_id);<br><br>-- COMMAND ----------<br><br>-- -- Create the dim_host table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_host(<br>    host_uri_id BIGINT NOT NULL PRIMARY KEY,<br>    host_uri STRING ,<br>    etl_date Timestamp <br>)<br>USING delta;<br><br>-- COMMAND ----------<br><br>-- -- Create the dim_client_type table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_client_type (<br>    client_type_id INT NOT NULL PRIMARY KEY ,  <br>    client_type_name STRING,      <br>    etl_Date TIMESTAMP<br>) <br>USING DELTA;<br><br>-- COMMAND ----------<br><br>-- -- Create the dim_device_name table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_device_name (<br>    device_name_id INT NOT NULL PRIMARY KEY,           <br>    device_name STRING,          <br>    etl_Date TIMESTAMP<br>) <br>USING DELTA;<br><br>-- COMMAND ----------<br><br>-- Create the dim_carrier_type table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_carrier_type (<br>    carrier_id INT NOT NULL PRIMARY KEY,               <br>    carrier_name STRING,          <br>    etl_Date TIMESTAMP<br>) <br>USING DELTA;  <br><br>-- COMMAND ----------<br><br>-- Create the dim_call_letter_name table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_call_letter_name (<br>    call_letter_name_id INT NOT NULL PRIMARY KEY,       <br>    call_letter_name STRING,       <br>    etl_Date TIMESTAMP<br>) <br>USING DELTA;<br><br>-- COMMAND ----------<br><br>-- Create the dim_player_type_name table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_player_type_name (<br>    player_type_id INT NOT NULL PRIMARY KEY,             <br>    player_type_name STRING,        <br>    etl_Date TIMESTAMP<br>) <br>USING DELTA; <br><br>-- Create the dim_cdn table with specified columns and data types.<br>-- Using delta.isolationLevel&#39; = &#39;Serializable&#39; to protect against concurrent inserts adding same row.<br>CREATE TABLE IF NOT EXISTS silver.dim_cdn<br>        (cdn_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,<br>        cdn_name STRING NOT NULL,<br>        etl_date TIMESTAMP NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (cdn_id)<br>    TBLPROPERTIES (&#39;delta.isolationLevel&#39; = &#39;Serializable&#39;);<br><br>INSERT INTO silver.dim_cdn (cdn_id, cdn_name, etl_date)<br>    VALUES (0, &#39;Unknown&#39;, current_timestamp());<br><br>-- Create the dim_distributor table with specified columns and data types.<br>-- Using delta.isolationLevel&#39; = &#39;Serializable&#39; to protect against concurrent inserts adding same row.<br>CREATE TABLE IF NOT EXISTS silver.dim_distributor<br>        (distributor_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,<br>        distributor_name STRING NOT NULL,<br>        etl_date TIMESTAMP NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (distributor_id)<br>    TBLPROPERTIES (&#39;delta.isolationLevel&#39; = &#39;Serializable&#39;);<br><br>INSERT INTO silver.dim_distributor (distributor_id, distributor_name, etl_date)<br>    VALUES (0, &#39;Unknown&#39;, current_timestamp());<br><br>-- Create the dim_episode table with specified columns and data types.<br>-- Using delta.isolationLevel&#39; = &#39;Serializable&#39; to protect against concurrent inserts adding same row.<br>CREATE TABLE IF NOT EXISTS silver.dim_episode<br>        (episode_id STRING NOT NULL,<br>        provided_episode_id STRING NOT NULL,<br>        episode_title STRING NOT NULL,<br>        episode_published_date TIMESTAMP NOT NULL,<br>        episode_duration_sec BIGINT NOT NULL,<br>        etl_date TIMESTAMP NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (episode_id)<br>    TBLPROPERTIES (&#39;delta.isolationLevel&#39; = &#39;Serializable&#39;);<br><br>INSERT INTO silver.dim_episode (episode_id, provided_episode_id, episode_title, episode_published_date, episode_duration_sec, etl_date)<br>    VALUES (&#39;0&#39;, &#39;Unknown&#39;, &#39;Unknown&#39;, &#39;1900-01-01&#39;, 0, current_timestamp());<br><br>-- Create the dim_program table with specified columns and data types.<br>-- Using delta.isolationLevel&#39; = &#39;Serializable&#39; to protect against concurrent inserts adding same row.<br>CREATE TABLE IF NOT EXISTS silver.dim_program<br>        (program_id BIGINT NOT NULL,<br>        program_title STRING NOT NULL,<br>        category_name STRING NOT NULL,<br>        etl_date TIMESTAMP NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (program_id)<br>    TBLPROPERTIES (&#39;delta.isolationLevel&#39; = &#39;Serializable&#39;);<br><br>INSERT INTO silver.dim_program (program_id, program_title, category_name, etl_date)<br>    VALUES (0, &#39;Unknown&#39;, &#39;Unknown&#39;, current_timestamp());<br><br>-- Create the dim_file_name table with specified columns and data types.<br>-- Using delta.isolationLevel&#39; = &#39;Serializable&#39; to protect against concurrent inserts adding same row.<br>CREATE TABLE IF NOT EXISTS silver.dim_file_name<br>        (file_name_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,<br>        file_name STRING NOT NULL,<br>        etl_date TIMESTAMP NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (file_name_id)<br>    TBLPROPERTIES (&#39;delta.isolationLevel&#39; = &#39;Serializable&#39;);<br><br>INSERT INTO silver.dim_file_name (file_name_id, file_name, etl_date)<br>    VALUES (0, &#39;Unknown&#39;, current_timestamp());<br><br>-- Create the silver.triton_podcast table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS silver.triton_podcast<br>        (dt DATE NOT NULL,<br>        session_start_date TIMESTAMP NOT NULL,<br>        pcm_listener_id STRING NOT NULL,<br>        triton_listener_id STRING,<br>        ip_address STRING NOT NULL,<br>        is_ip_valid BOOLEAN NOT NULL,<br>        user_agent STRING,<br>        mobile_advertising_id STRING,<br>        is_maid_valid BOOLEAN NOT NULL,<br>        cdn_id BIGINT NOT NULL,<br>        distributor_id BIGINT NOT NULL,<br>        program_id BIGINT NOT NULL,<br>        episode_id STRING NOT NULL,<br>        unique_content_served_duration_sec BIGINT NOT NULL,<br>        is_ivt BOOLEAN NOT NULL,<br>        ivt_type ARRAY&lt;STRING&gt; NOT NULL,<br>        etl_date TIMESTAMP NOT NULL,<br>        file_name_id BIGINT NOT NULL)<br>    USING DELTA<br>    PARTITIONED BY (dt);<br><br>-- COMMAND ----------<br><br>-- Create the dim_played_from table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS dim_played_from (<br>    played_from_value INT NOT NULL PRIMARY KEY, <br>    played_from_name_classification STRING<br>)<br>USING DELTA;<br><br>-- Create the transunion_id_resolution_response_history table with specified columns and data types.<br>CREATE TABLE IF NOT EXISTS silver.transunion_id_resolution_response_history<br>        (id_resolution_attempt_id BIGINT NOT NULL,<br>        extern_tuid STRING,<br>        extern_tuhhid STRING,<br>        response_date TIMESTAMP NOT NULL,<br>        etl_date TIMESTAMP NOT NULL,<br>        file_name_id BIGINT NOT NULL)<br>    USING DELTA<br>    CLUSTER BY (id_resolution_attempt_id);</pre><p>landing:</p><pre>etl scripts:<br><br><br>landing scripts<br>dim_profile_landing<br><br>from datetime import datetime, timedelta<br>from pytz import timezone<br>from pyspark.sql.functions import col, row_number,lit<br>from pyspark.sql.window import Window<br>from pyspark.sql.types import DateType<br>import re<br><br># job parameters<br>landing_schema_name = dbutils.widgets.get(&quot;landing_schema_name&quot;)<br>landing_table_name = dbutils.widgets.get(&quot;landing_table_name&quot;)<br>s3_path = dbutils.widgets.get(&quot;s3_path&quot;)<br>n_days = int(dbutils.widgets.get(&quot;n_days&quot;))<br><br># change the timezone to your local timezone<br>tz = timezone(&quot;America/Chicago&quot;)  <br>current_date = datetime.now(tz) - timedelta(1)  # Yesterday&#39;s date<br>start_date = current_date - timedelta(days=n_days - 1)<br><br># Generate the list of file paths dynamically<br>file_paths = []<br>while start_date &lt;= current_date:<br>   path = f&quot;{s3_path}/dt={start_date.strftime(&#39;%Y-%m-%d&#39;)}/&quot;<br>   file_paths.append(path)<br>   start_date += timedelta(days=1)<br><br># Create an empty list to store DataFrames with the added date column<br>df_list = []<br><br># Load data from the generated file paths<br>for path in file_paths:<br>    # Extract the date from the file path using regex (assumes dt=YYYY-MM-DD format)<br>    date_match = re.search(r&#39;dt=(\d{4}-\d{2}-\d{2})&#39;, path)<br>    if date_match:<br>        dt = date_match.group(1)  # Extracted date as string<br>    <br>        # Read the data from the Avro files<br>        temp_df = spark.read.format(&#39;avro&#39;).load(path)<br>        <br>        # Add a new column with the extracted date<br>        temp_df = temp_df.withColumn(&quot;dt&quot;, lit(dt).cast(DateType()))<br>        <br>        # Append the DataFrame to the list<br>        df_list.append(temp_df)<br>    else:<br>        # Handle the case where no date is found in the file path<br>        print(f&quot;No date found in the path: {path}&quot;)<br><br>        # append the original DataFrame with a default date (e.g., &#39;1970-01-01&#39;)<br>        temp_df = spark.read.format(&#39;avro&#39;).load(path)<br>        temp_df = temp_df.withColumn(&quot;dt&quot;, lit(&quot;1970-01-01&quot;).cast(DateType()))<br>        df_list.append(temp_df)    <br><br># Combine all DataFrames into one<br>final_df = df_list[0]<br>for df in df_list[1:]:<br>    final_df = final_df.union(df)<br><br>#define a window specification to partition by profile_id and order by dt in descending order<br>window_spec = Window.partitionBy(&quot;profile_id&quot;).orderBy(col(&quot;dt&quot;).desc())<br><br># Add row number to identify the latest record for each profile_id<br>final_df = final_df.withColumn(&quot;row_number&quot;, row_number().over(window_spec))<br><br># Filter to keep only the latest record (row_number = 1)<br>final_df = final_df.filter(col(&quot;row_number&quot;) == 1).drop(&quot;row_number&quot;)<br><br># Truncate the table before appending<br>spark.sql(f&quot;truncate table {landing_schema_name}.{landing_table_name}&quot;)<br><br># Write data to the Delta table in append mode<br>final_df.write.mode(&#39;append&#39;).saveAsTable(f&quot;{landing_schema_name}.{landing_table_name}&quot;)<br><br>print(f&quot;Successfully loaded data for the past {n_days} days.&quot;)</pre><p>workflow:</p><pre>Workflow<br><br><br>task 1:<br><br>load_transunion_response_from_s3_to_landing  --&gt;   etl_scripts/landing/transunion_response.landing.load<br><br>path : https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542066?o=954740910440412#command/3467273560554802<br><br><br>%pip install -r /Workspace/Shared/Shared_Libraries/shared_library_dependencies_requirements.txt -q<br><br><br>&quot;&quot;&quot;<br>Description:<br>This script reads data from S3, processes it, and writes it to Delta tables in Databricks. <br>It uses widgets to parameterize the S3 folder path, loads the data from CSV files into DataFrames, <br>adds metadata columns, and saves the transformed data to Delta tables in the landing zone. <br>After processing, it deletes the source files from the S3 folder.<br><br>Steps:<br>1. Create a Databricks widget for the S3 folder path.<br>2. Retrieve the S3 folder path from the widget.<br>3. Load CSV files from the S3 folder into DataFrames with the quote option.<br>4. Add metadata columns to the DataFrames.<br>5. Write the DataFrames to Delta tables in the landing zone.<br>6. Delete the source files from the S3 folder.<br>&quot;&quot;&quot;<br><br># Import necessary modules and functions<br>from pyspark.sql.functions import col, regexp_extract<br><br># Step 1: Create a widget for the S3 folder path<br># dbutils.widgets.text(&quot;transunion_s3_folder&quot;, &quot;s3a://triton-audience-platform-delta-lake-dev/from_transunion_decrypted/&quot;, &quot;S3_Folder_Path&quot;)<br><br># Step 2: Retrieve the job parameters<br>transunion_s3_folder = dbutils.widgets.get(&quot;transunion_s3_folder&quot;)<br><br># Step 3: Load all files from the decrypted folder to the landing table, adding metadata columns<br>landing_df_id_resolution_response = spark.read \<br>    .option(&quot;sep&quot;, &#39;|&#39;) \<br>    .option(&quot;header&quot;, True) \<br>    .option(&quot;quote&quot;, &quot;\&quot;&quot;) \<br>    .csv(transunion_s3_folder + &quot;2453_10000411_OUTPUT_D*_Resolution.psv&quot;) \<br>    .withColumn(&quot;response_date&quot;, col(&quot;_metadata.file_modification_time&quot;)) \<br>    .withColumn(&quot;file_name&quot;, col(&quot;_metadata.file_name&quot;))<br><br>landing_df_demographic = spark.read \<br>    .option(&quot;sep&quot;, &#39;|&#39;) \<br>    .option(&quot;header&quot;, True) \<br>    .option(&quot;quote&quot;, &quot;\&quot;&quot;) \<br>    .csv(transunion_s3_folder + &quot;2453_10000411_OUTPUT_D*_Appends.psv&quot;) \<br>    .withColumn(&quot;response_date&quot;, col(&quot;_metadata.file_modification_time&quot;)) \<br>    .withColumn(&quot;file_name&quot;, col(&quot;_metadata.file_name&quot;))<br><br># Step 4: Write the DataFrames to Delta tables in the landing zone<br>landing_df_id_resolution_response.write.format(&quot;delta&quot;).mode(&#39;overwrite&#39;).saveAsTable(&quot;landing.transunion_id_resolution_response&quot;)<br>landing_df_demographic.write.format(&quot;delta&quot;).mode(&#39;overwrite&#39;).saveAsTable(&quot;landing.transunion_demographic&quot;)<br><br># Step 5: Delete source files from the S3 folder by removing the entire directory and recreating it<br>dbutils.fs.rm(transunion_s3_folder, True)<br>dbutils.fs.mkdirs(transunion_s3_folder)<br><br><br>task 2 :<br><br>etl_transunion_id_response_history_from_landing_to_silver --&gt; etl_scripts/silver/transunion_id_resolution_response_history.silver.etl<br><br>https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542062?o=954740910440412#command/3467273560554804<br><br>%pip install -r /Workspace/Shared/Shared_Libraries/shared_library_dependencies_requirements.txt -q<br><br>from pyspark.sql.functions import *<br>from sparketlutillib.sparketlutil import SparkETLUtil<br>spark_etl_util = SparkETLUtil(spark)<br><br># Read the new/updated data from the landing zone table and save it to a dataframe.<br>landing_tu_id_response_df = spark.read.table(&quot;landing.transunion_id_resolution_response&quot;)<br><br># Clean the data from the landing zone and rename any source columns before adding calculated columns.<br>clean_tu_id_response_df = landing_tu_id_response_df \<br>    .withColumn(&quot;id_resolution_attempt_id&quot;, col(&quot;CustomerID&quot;).cast(&#39;long&#39;)) \<br>    .withColumnRenamed(&quot;Extern_TUID&quot;, &quot;extern_tuid&quot;) \<br>    .withColumnRenamed(&quot;Extern_TUHHID&quot;, &quot;extern_tuhhid&quot;) \<br>    .drop(&quot;CustomerID&quot;)<br><br># Populate the dim_file_name table with any new file names since last job run.<br>merge_dim_file_name_df = clean_tu_id_response_df \<br>    .select(&quot;file_name&quot;) \<br>    .distinct() \<br>    .withColumn(&quot;etl_date&quot;, current_timestamp())<br><br>dim_file_name_df = spark_etl_util.merge_dimension_table(merge_dim_file_name_df, &quot;silver.dim_file_name&quot;, [&quot;file_name&quot;], [&quot;file_name_id&quot;, &quot;etl_date&quot;])<br><br># Create the final dataframe with same schema as the target table, joining to add any dimension table IDs.<br>joined_tu_id_response_df = clean_tu_id_response_df \<br>    .join(dim_file_name_df, on=&quot;file_name&quot;, how=&#39;left_outer&#39;) \<br>    .select(&quot;id_resolution_attempt_id&quot;, &quot;extern_tuid&quot;, &quot;extern_tuhhid&quot;, &quot;response_date&quot;, &quot;file_name_id&quot;) \<br>    .withColumn(&quot;etl_date&quot;, current_timestamp()) \<br><br>joined_tu_id_response_df.write.saveAsTable(&quot;silver.transunion_id_resolution_response_history&quot;, mode=&#39;append&#39;)<br><br><br><br><br><br><br><br>task3 :<br><br>update_id_resolution_send_progress_table_silver  ---&gt; etl_scripts/others/update_id_resolution_send_progress_table<br><br>https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542060?o=954740910440412#command/3467273560554805<br><br><br># Step 1: Update matched records (set &#39;M&#39; for matched CustomerID)<br>spark.sql(&quot;&quot;&quot;<br>MERGE INTO silver.id_resolution_send_progress AS sp<br>USING landing.transunion_id_resolution_response AS r<br>ON sp.id_resolution_attempt_id = r.CustomerID<br>WHEN MATCHED THEN<br>    UPDATE SET sp.id_resolution_status_cd = &#39;M&#39;<br>&quot;&quot;&quot;)<br><br># Step 2: Update unmatched records (set &#39;N&#39; for unmatched CustomerID)<br>spark.sql(&quot;&quot;&quot;<br>UPDATE silver.id_resolution_send_progress<br>SET id_resolution_status_cd = &#39;N&#39;<br>WHERE id_resolution_attempt_id NOT IN (<br>    SELECT CustomerID FROM landing.transunion_id_resolution_response<br>)<br>&quot;&quot;&quot;)<br><br><br><br><br>task4 : load_pord_triton_individual_tbl ---&gt; etl_scripts/prod/triton_individual_prod<br><br>https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542035?o=954740910440412#command/3467273560554810<br><br>&quot;&quot;&quot;<br>This script handles two scenarios for managing records in the prod.triton_individual table:<br><br>1. **Inserting New Records:**<br>   - Insert new records into the `prod.triton_individual_prod` table for `TU_ID`s that are **not already present** in the `id_res_link` table.<br>   - This ensures that any new `TU_ID`s coming from external sources (e.g., a partner system) will be added to the `prod.triton_individual_prod` table for further processing or linkage.<br><br>2. **Updating Existing Records:**<br>   - For `TU_ID`s that already exist in the `prod.triton_individual` table, the script will **update the existing records**.<br>   - The following fields will be updated for the matching `TU_ID` records: `email`, `phone`, `latest_ipaddress`, `user_agent`, `terminal_id`, `name`, `birth_year`, `gender_cd`.<br>   - This ensures the `prod.triton_individual` table has the latest and most accurate information for existing users, including personal and contact information, as well as additional user data (name, birth year, gender, IP address, user agent, and terminal ID).<br>&quot;&quot;&quot;<br>-----------<br><br><br>import uuid<br>from pyspark.sql.window import Window<br>from pyspark.sql.types import StringType<br>from delta.tables import DeltaTable<br>from pyspark.sql.functions import col, when,current_timestamp,coalesce,udf<br><br># Define a UDF to generate UUID for each row (unique per record)<br>def generate_uuid():<br>    return str(uuid.uuid4())<br><br># Register the UDF with Spark<br>generate_uuid_udf = udf(generate_uuid, StringType())<br><br><br>-----------<br><br><br># 1. **Inserting New Records:**<br># -----------------------------<br># This section handles inserting new records into the `prod.triton_individual` table<br># where the `tu_id` is not present in the `individual_id_res_link` table.<br><br>final_df=spark.sql(f&quot;&quot;&quot;        <br> with matched_dataset AS (<br>    -- create a matched dataset by joining the &#39;send_progress&#39; and &#39;response&#39; tables.<br>    SELECT send_progress.*, response.*<br>    FROM silver.id_resolution_send_progress send_progress<br>    JOIN landing.transunion_id_resolution_response response<br>        ON send_progress.id_resolution_attempt_id = response.CustomerID<br>    WHERE response.Extern_TUID IS NOT NULL <br>)<br><br>-- select distinct records for new entries where the TUID is not already linked<br>SELECT distinct link.triton_individual_guid,link.id_resolution_link_id as id_resolution_link_id, <br>dim.name as name,<br>matched.email_address as email_address,<br>matched.phone_number as phone_number,<br>dim.birth_year as birth_year,<br>coalesce(dim.gender_cd,&#39;U&#39;) as gender_cd,<br>current_timestamp() as triton_individual_create_date,<br>matched.id_resolution_attempt_id as created_by,<br>NULL as last_ip_address,<br>NULL as last_mobile_advertising_id,<br>NULL as last_user_agent,<br>NULL as last_terminal_id<br>FROM matched_dataset matched<br>LEFT JOIN prod.individual_id_res_link link<br>    ON matched.extern_tuid = link.id_resolution_link_id<br>    AND link.id_resolution_link_type_cd = &#39;TU&#39;<br>    --LEFT JOIN with the `dim_profile` table to add additional profile details based on &#39;source_id&#39;<br>LEFT JOIN silver.dim_profile dim <br>    ON matched.source_id = dim.profile_id <br>             &quot;&quot;&quot;)<br><br># Filter records where &#39;id_resolution_link_id&#39; is NULL (new records).<br>df_new_records = final_df.filter(final_df.id_resolution_link_id.isNull())<br><br># Filter records where &#39;id_resolution_link_id&#39; is NOT NULL (existing records).<br>df_existed_records = final_df.filter(final_df.id_resolution_link_id.isNotNull())<br><br># Add UUID and ETL date columns, then select only the required columns for the final insert<br>triton_individual_records_df = (df_new_records<br>                                .withColumn(&quot;triton_individual_guid&quot;, generate_uuid_udf())<br>                                .withColumn(&quot;etl_date&quot;, current_timestamp())<br>                                .select(&quot;triton_individual_guid&quot;, &quot;name&quot;, &quot;email_address&quot;, &quot;phone_number&quot;, &quot;birth_year&quot;, &quot;gender_cd&quot;, &quot;last_ip_address&quot;, &quot;last_user_agent&quot;, &quot;last_mobile_advertising_id&quot;, &quot;last_terminal_id&quot;, &quot;triton_individual_create_date&quot;, &quot;created_by&quot;, &quot;etl_date&quot;))<br><br><br># Insert the new records into the target Delta table &#39;prod.triton_individual&#39;<br>triton_individual_records_df.write.format(&quot;delta&quot;).mode(&quot;append&quot;).saveAsTable(&quot;prod.triton_individual&quot;)<br><br><br><br>--------<br><br># 2. **Updating Existing Records:**<br># ----------------------------------<br><br># Step 1: Load the existing data into a DeltaTable object (prod.triton_individual)<br>delta_table = DeltaTable.forName(spark, &quot;prod.triton_individual&quot;)<br><br># Step 2: Remove duplicates from the Delta table based on the column &#39;triton_individual_guid&#39;<br>df_match_unique = df_existed_records.dropDuplicates([&quot;triton_individual_guid&quot;])<br><br># Step 3:Perform a merge between the Delta table and the cleaned DataFrame (df_match_unique)<br>delta_table.alias(&quot;t&quot;).merge(<br>    df_match_unique.alias(&quot;s&quot;),  <br>    &quot;t.triton_individual_guid = s.triton_individual_guid&quot;  # Match condition on &#39;triton_individual_guid&#39;<br>).whenMatchedUpdate(<br>    condition = (<br>        # Check if any of the columns differ between the source and target data<br>        # If a difference is found, update the record<br>        (col(&quot;s.email_address&quot;) != col(&quot;t.email_address&quot;)) |<br>        (col(&quot;s.phone_number&quot;) != col(&quot;t.phone_number&quot;)) |<br>        (col(&quot;s.last_ip_address&quot;) != col(&quot;t.last_ip_address&quot;)) |<br>        (col(&quot;s.last_mobile_advertising_id&quot;) != col(&quot;t.last_mobile_advertising_id&quot;)) |<br>        (col(&quot;s.last_user_agent&quot;) != col(&quot;t.last_user_agent&quot;)) |<br>        (col(&quot;s.last_terminal_id&quot;) != col(&quot;t.last_terminal_id&quot;)) |<br>        (col(&quot;s.name&quot;) != col(&quot;t.name&quot;)) |<br>        (col(&quot;s.birth_year&quot;) != col(&quot;t.birth_year&quot;)) |<br>        (col(&quot;s.gender_cd&quot;) != col(&quot;t.gender_cd&quot;))<br>    ),  <br>    set = {<br>        # Set the updated values from the source DataFrame for the matching columns<br>        &quot;email_address&quot;: &quot;s.email_address&quot;,<br>        &quot;phone_number&quot;: &quot;s.phone_number&quot;,<br>        &quot;last_ip_address&quot;: &quot;s.last_ip_address&quot;,<br>        &quot;last_mobile_advertising_id&quot;: &quot;s.last_mobile_advertising_id&quot;,<br>        &quot;last_user_agent&quot;: &quot;s.last_user_agent&quot;,<br>        &quot;last_terminal_id&quot;: &quot;s.last_terminal_id&quot;,<br>        &quot;name&quot;: &quot;s.name&quot;,  <br>        &quot;birth_year&quot;: &quot;s.birth_year&quot;,<br>        &quot;gender_cd&quot;: &quot;s.gender_cd&quot;,<br>        &quot;etl_date&quot;: current_timestamp()  <br>    }<br>).execute()<br><br><br><br>task5 :load_prod_individual_id_res_link_tbl---&gt; etl_scripts/prod/response_individual_id_res_link_prod<br><br>https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542051?o=954740910440412#command/2069367323081206<br>task6 :<br><br>etl_transunion_demographic_from_landing_to_silver ---&gt; etl_scripts/silver/transunion_demographic.silver.etl<br><br>https://dbc-d1e6f9d8-1cd0.cloud.databricks.com/editor/notebooks/1983667784542057?o=954740910440412#command/2069367323081211</pre><p>terraform workflow.wf</p><pre>{<br> &quot;job_id&quot;: 870708042318616,<br> &quot;creator_user_name&quot;: &quot;mike.barzilli@tritondigital.com&quot;,<br> &quot;run_as_user_name&quot;: &quot;mike.barzilli@tritondigital.com&quot;,<br> &quot;run_as_owner&quot;: true,<br> &quot;settings&quot;: {<br>  &quot;name&quot;: &quot;transunion_response.wf&quot;,<br>  &quot;description&quot;: &quot;This job executes notebooks to load and ETL results to landing.transunion_id_resolution_response in landing zone and then to silver.transunion_id_resolution_response_history table in the silver zone.\n\nThe first task reads data from a pipe separated file (psv) from the S3 inbound decrypted folder from TransUnion.\n\nThe second task reads data from the landing table transunion_id_resolution_response and writes it to the history table.&quot;,<br>  &quot;email_notifications&quot;: {<br>   &quot;on_failure&quot;: [<br>    &quot;audience_platform_job_alert@tritondigital.com&quot;<br>   ],<br>   &quot;on_duration_warning_threshold_exceeded&quot;: [<br>    &quot;audience_platform_job_alert@tritondigital.com&quot;<br>   ],<br>   &quot;no_alert_for_skipped_runs&quot;: false<br>  },<br>  &quot;webhook_notifications&quot;: {},<br>  &quot;notification_settings&quot;: {<br>   &quot;no_alert_for_skipped_runs&quot;: false,<br>   &quot;no_alert_for_canceled_runs&quot;: false<br>  },<br>  &quot;timeout_seconds&quot;: 7200,<br>  &quot;health&quot;: {<br>   &quot;rules&quot;: [<br>    {<br>     &quot;metric&quot;: &quot;RUN_DURATION_SECONDS&quot;,<br>     &quot;op&quot;: &quot;GREATER_THAN&quot;,<br>     &quot;value&quot;: 3600<br>    }<br>   ]<br>  },<br>  &quot;schedule&quot;: {<br>   &quot;quartz_cron_expression&quot;: &quot;33 0 10 * * ?&quot;,<br>   &quot;timezone_id&quot;: &quot;UTC&quot;,<br>   &quot;pause_status&quot;: &quot;PAUSED&quot;<br>  },<br>  &quot;max_concurrent_runs&quot;: 1,<br>  &quot;tasks&quot;: [<br>   {<br>    &quot;task_key&quot;: &quot;load_transunion_response_from_s3_to_landing&quot;,<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/landing/transunion_response.landing.load&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   },<br>   {<br>    &quot;task_key&quot;: &quot;etl_transunion_id_response_history_from_landing_to_silver&quot;,<br>    &quot;depends_on&quot;: [<br>     {<br>      &quot;task_key&quot;: &quot;load_transunion_response_from_s3_to_landing&quot;<br>     }<br>    ],<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/silver/transunion_id_resolution_response_history.silver.etl&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   },<br>   {<br>    &quot;task_key&quot;: &quot;update_id_resolution_send_progress_table_silver&quot;,<br>    &quot;depends_on&quot;: [<br>     {<br>      &quot;task_key&quot;: &quot;etl_transunion_id_response_history_from_landing_to_silver&quot;<br>     }<br>    ],<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/others/update_id_resolution_send_progress_table&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   },<br>   {<br>    &quot;task_key&quot;: &quot;load_pord_triton_individual_tbl&quot;,<br>    &quot;depends_on&quot;: [<br>     {<br>      &quot;task_key&quot;: &quot;update_id_resolution_send_progress_table_silver&quot;<br>     }<br>    ],<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/prod/triton_individual_prod&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   },<br>   {<br>    &quot;task_key&quot;: &quot;load_prod_individual_id_res_link_tbl&quot;,<br>    &quot;depends_on&quot;: [<br>     {<br>      &quot;task_key&quot;: &quot;load_pord_triton_individual_tbl&quot;<br>     }<br>    ],<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/prod/response_individual_id_res_link_prod&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   },<br>   {<br>    &quot;task_key&quot;: &quot;etl_transunion_demographic_from_landing_to_silver&quot;,<br>    &quot;depends_on&quot;: [<br>     {<br>      &quot;task_key&quot;: &quot;load_prod_individual_id_res_link_tbl&quot;<br>     }<br>    ],<br>    &quot;run_if&quot;: &quot;ALL_SUCCESS&quot;,<br>    &quot;notebook_task&quot;: {<br>     &quot;notebook_path&quot;: &quot;etl_scripts/silver/transunion_demographic.silver.etl&quot;,<br>     &quot;source&quot;: &quot;GIT&quot;<br>    },<br>    &quot;timeout_seconds&quot;: 0,<br>    &quot;email_notifications&quot;: {},<br>    &quot;webhook_notifications&quot;: {}<br>   }<br>  ],<br>  &quot;git_source&quot;: {<br>   &quot;git_url&quot;: &quot;https://github.com/triton-digital-em/audience-platform-databricks.git&quot;,<br>   &quot;git_provider&quot;: &quot;gitHub&quot;,<br>   &quot;git_branch&quot;: &quot;main&quot;<br>  },<br>  &quot;format&quot;: &quot;MULTI_TASK&quot;,<br>  &quot;queue&quot;: {<br>   &quot;enabled&quot;: true<br>  },<br>  &quot;parameters&quot;: [<br>   {<br>    &quot;name&quot;: &quot;transunion_s3_folder&quot;,<br>    &quot;default&quot;: &quot;s3a://triton-audience-platform-delta-lake-dev/from_transunion_decrypted/&quot;<br>   }<br>  ]<br> },<br> &quot;created_time&quot;: 1735003310113<br>}</pre><p>marr:</p><pre>seq.to ds.todf<br><br>https://git.marriott.com/RMS/RateETL/blob/feature-mgp/RMMGP-43548/PricePlanIngestion/src/test/scala/com.marriott.mgpapp.spark.priceplan.test/helper/PricePlanIngestionHelperTest.scala<br><br>this is main commit :<br><br>https://git.marriott.com/RMS/RateETL/commit/591ecf5eab90ba5c9422dc1994450114b41554e5<br><br><br> val genTimeDataset = Seq(<br>          (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>        ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>        val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br><br><br><br><br>=============================================================================================<br><br>        medium article<br><br><br>empty datfame<br>utkarsha<br><br><br><br> val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br><br>      // Create an empty DataFrame with the same schema as recoverableDocDBDF<br>      val schema = recoverableDocDBDS.schema<br>      val emptyDataFrame = sparkSession.createDataFrame(sparkSession.sparkContext.emptyRDD[Row], schema)<br><br>      (emptyDataset, emptyDataFrame)<br><br><br>      ==================================================================================================================<br><br>      why do we use Seq explain with simple example whts is it need here<br><br>val validRow = Row(<br>      &quot;&quot;,<br>      &quot;aoppor&quot;,<br>      &quot;aquote&quot;,<br>      &quot;Create&quot;,<br>      &quot;2023-12-26T23:59:54.000Z&quot;,<br>      &quot;13295284&quot;,<br>      &quot;MEMWI&quot;,<br>      &quot;GA4&quot;,<br>      &quot;Group Contract Name&quot;,<br>      &quot;Convention&quot;,<br>      &quot;2023-12-23&quot;,<br>      &quot;2023-12-25&quot;,<br>      &quot;2023-12-20T23:59:59&quot;,<br>      &quot;2023-12-26T23:59:59&quot;,<br>      &quot;KING&quot;,<br>      null,<br>      &quot;2019-11-22&quot;,<br>      &quot;2019-12-20&quot;,<br>      &quot;C&quot;,<br>      &quot;110&quot;,<br>      &quot;USD&quot;,<br>      &quot;0&quot;,<br>      &quot;0&quot;,<br>      &quot;MEMWIGA413295284&quot;<br>    )<br>    // Define SparkSession for testing<br>    val sparkSS: SparkSession =<br>      SparkSession.builder.appName(&quot;sample&quot;).master(local).getOrCreate()<br>    val row = sparkSS.sparkContext.parallelize(Seq(validRow))<br> <br> <br> why case class data is converted to sequence first anf then dataframe , why not directly to dataframe<br> <br> <br> Converting case class data to a DataFrame in Apache Spark typically involves an intermediate step of converting the data to a sequence or collection. This intermediate step is often taken to leverage certain functionalities and ensure type safety. Here&#39;s a more detailed breakdown of why this process is commonly used:<br><br>1. Type Safety and Schema Inference<br><br>Case Class Definition: A case class in Scala allows you to define a schema with fields and types. This is useful because it provides a clear and strongly-typed schema.<br>Sequence Conversion: When you convert a collection of case class instances (like Seq[MyCaseClass]) to a DataFrame, Spark can automatically infer the schema based on the case class definition. This means that you get a DataFrame with the correct column names and types without having to manually define the schema.<br><br><br>2. Ease of Use and Readability<br><br>Intermediate Sequence: Converting to a sequence makes it easier to handle and manipulate the data before creating the DataFrame. This intermediate step allows for transformations, filtering, and other operations using Scala&#39;s rich collection API.<br>Direct Conversion: While it is possible to directly convert raw data (like RDDs) to a DataFrame, doing so usually requires additional steps for schema definition and might not be as straightforward or readable.<br><br><br>3. Performance Considerations<br><br>Optimized Execution: Spark optimizes the execution of transformations and actions. By converting to a sequence and then to a DataFrame, you allow Spark to better optimize the operations on the data, leveraging its Catalyst optimizer for efficient execution.<br><br><br>4. Flexibility<br><br>Data Manipulation: By converting to a sequence first, you have more flexibility to manipulate the data using Scala&#39;s powerful collection operations before converting it to a DataFrame. This can be particularly useful for preprocessing or cleaning the data.<br><br><br>Example Code<br>Here&#39;s an example to illustrate the process:<br><br>// Define a case class<br>case class Person(name: String, age: Int)<br><br>// Create a sequence of case class instances<br>val peopleSeq = Seq(<br>  Person(&quot;Alice&quot;, 29),<br>  Person(&quot;Bob&quot;, 31),<br>  Person(&quot;Charlie&quot;, 25)<br>)<br><br>// Convert the sequence to a DataFrame<br>import spark.implicits._<br>val peopleDF = peopleSeq.toDF()<br><br>// Show the DataFrame<br>peopleDF.show()<br>In this example:<br><br>A case class Person is defined.<br>A sequence of Person instances is created.<br>The sequence is then converted to a DataFrame using the toDF method, which automatically infers the schema from the case class.<br><br><br>Direct Conversion (Alternative)<br>While it is less common, you can directly create a DataFrame from an RDD of case class instances:<br><br>val peopleRDD = spark.sparkContext.parallelize(peopleSeq)<br>val peopleDF = spark.createDataFrame(peopleRDD)<br>peopleDF.show()<br>This method is less straightforward and might not leverage all the benefits of type safety and schema inference as effectively as the sequence-to-DataFrame approach.<br><br>Conclusion<br>The common practice of converting case class data to a sequence before creating a DataFrame is due to reasons of type safety, ease of use, performance optimization, and flexibility. While direct conversion is possible, the intermediate sequence step often provides significant advantages.<br><br><br>Copy<br>Thank you for your feedback.<br>Icon<br><br><br>==============================================collation==================================================================================<br><br><br>  def genarateEEHData(sparkSession: SparkSession): DataFrame = {<br>    import sparkSession.implicits._<br>    sparkSession.createDataset(List(TestsHelper.toString)).as(&quot;value&quot;).toDF()<br>  }<br><br><br><br>\<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>  ====================================tag===============================================<br>  Tag: Used to mark specific rows and identify table instances.<br>AbstractTable: Provides the framework for defining table schemas.<br>Usage: Tags are typically used internally by Slick to manage and differentiate between table instances and their rows.<br>In essence, a Tag is a vital part of Slick&#39;s internal mechanism for managing database tables and their rows, ensuring that each table instance is uniquely identifiable.<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>============================================row , tuple----------------------------------------------------------------------------------------<br><br>==========================================================pick later==========================================================<br>        this is the main method<br>          def getProductCatalogToPropertyDataFrame(parsedDF: DataFrame, bucketName: String)(implicit sparkSession: SparkSession, dbDetails: DbDetails, dbMap: Map[String, String], schemaName: String) = {<br><br>    log.info(&quot;Started Transformations and Processing of the Data | getProductCatalogToPropertyDataFrame()&quot;, customMeta)<br><br>    try {<br><br>      val activePostGresRecords = getPreviousDaySnapshot(dbDetails, dbMap, schemaName)<br><br>      // Drop the columns which are not required.<br>      val postGresDataDF = activePostGresRecords<br>        .drop(PcptSk,<br>          PcptEffEndDate,<br>          PcptActiveInd,<br>          PcptStatusDate,<br>          PcptInsertedBy,<br>          PcptInsertedOn)<br><br>      import sparkSession.implicits._<br>      val parsedProductCatalogDataDF = parsedDF<br>        .drop(DeflaggedDate,<br>          PeopleSftId,<br>          Province,<br>          Market,<br>          MrwdsParticipate,<br>          NonbrandedFlag,<br>          OpsRegion,<br>          GlbDivision,<br>          GlbRegion,<br>          AvpCode,<br>          AvpDescription,<br>          AkCategory,<br>          PrestigeTierCat,<br>          SeasStartDt1,<br>          SeasEndDt1,<br>          SeasDiscSch1,<br>          SeasDiscPert1,<br>          SeasRecId1,<br>          SeasSortId1,<br>          SeasStartDt2,<br>          SeasEndDt2,<br>          SeasDiscSch2,<br>          SeasDiscPert2,<br>          SeasRecId2,<br>          SeasSortId2,<br>          SeasStartDt3,<br>          SeasEndDt3,<br>          SeasDiscSch3,<br>          SeasDiscPert3,<br>          SeasRecId3,<br>          SeasSortId3,<br>          SeasStartDt4,<br>          SeasEndDt4,<br>          SeasDiscSch4,<br>          SeasDiscPert4,<br>          SeasRecId4,<br>          SeasSortId4,<br>          SeasStartDt5,<br>          SeasEndDt5,<br>          SeasDiscSch5,<br>          SeasDiscPert5,<br>          SeasRecId5,<br>          SeasSortId5,<br>          CityTaxPert,<br>          CityTaxType,<br>          VatTaxPert,<br>          VatTaxType,<br>          Filler)<br>        .map(row =&gt; applyschema(row))<br><br>      //  Changing the precision of the decimal columns<br>      val decimalType = DataTypes.createDecimalType(7, 2)<br><br>      val productCatalogDF = parsedProductCatalogDataDF<br>        .withColumn(LatSeconds + &quot;_TMP&quot;, col(LatSeconds).cast(decimalType))<br>        .withColumn(LonSeconds + &quot;_TMP&quot;, col(LonSeconds).cast(decimalType))<br>        .drop(LatSeconds, LonSeconds)<br>        .withColumnRenamed(LatSeconds + &quot;_TMP&quot;, LatSeconds)<br>        .withColumnRenamed(LonSeconds + &quot;_TMP&quot;, LonSeconds)<br>        .withColumn(OWNER, substring(col(OWNER), 1, 40))<br>        .withColumn(OWNER, regexp_replace(col(OWNER), &quot;,&quot;, &quot; &quot;))<br><br>      // Filtering out unwanted product codes.<br>      val requiredPropertiesDF = productCatalogDF.filter(!(col(ProductCode) isin(&quot;ST&quot;, &quot;GX&quot;)))<br><br>      //    Joining(Left Outer) current data with the postgreSQL db data on property number<br>      //    and dropping columns which are newly created in the current process<br>      val leftJoinedDFWithActiveRecDF = requiredPropertiesDF.join(postGresDataDF,<br>        requiredPropertiesDF(PropertyNum) === postGresDataDF(PcptPropertyNum) and<br>          requiredPropertiesDF(PropertyCode) === postGresDataDF(PcptPropertyCode), &quot;left_outer&quot;)<br><br>      // Getting the records which have undergone changes with respect to the database records.<br>      val updateDF = getUpdateRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Getting the records which does not match with database records that is which are new records.<br>      val insertDF = getInsertRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Getting the delete records.<br>      val deleteDF = getDeteteRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Appending all the update,insert and delete records and adding few generic columns.<br>      val actionableDF = getUnionUpdateInsertDeleteDFs(updateDF, insertDF, deleteDF)<br><br>      // Dropping the database columns from the dataframe.<br>      val requiredColumnsDF = getRequiredColumnsDF(actionableDF)<br><br>      //    Renaming the input file columns to the database columns and creating the Json record<br>      //    which will be pushed to kafka required for ProductCatalogPropertyUpdateTargetJob*/<br>      val finalDataDF = getFinalDataFrame(requiredColumnsDF)<br><br>      //    Adding IsRecoverable column to differentiate recoverable, irrecoverable and valid data in the end to push to respective topics<br>      //    Making Recoverable as default, so that any exception occurs while dbconnection all the records will be pushed to error topic<br>      val persistDF = finalDataDF.withColumn(IsRecoverable, lit(Recoverable))<br><br>      //    Persisting the dataframe to database and respective topics<br>      persistDataFrame(persistDF, checkPointDirectory, bucketName, jobName)<br>    }<br>    catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;getProductCatalogToPropertyDataFrame() :: Exception while interacting with DB :: Exception: &quot; + ex.printStackTrace(), customMeta)<br>        val recoverableDF = parsedDF.coalesce(5)<br>          .select(from_json(col(SourceString), Encoders.product[EEHData].schema).as(Value))<br>        produceRecoverableMessage(recoverableDF)<br>    }<br><br>  }<br><br>   THIS IS THE MODIFIED TEST CASE WITH UPDATED SCHEMA FOR THIS<br><br>    test(&quot;getFinalDataFrame should return DataFrame with prefixed columns and JSON data column&quot;) {<br>    // Mock data for testing<br>    val spark: SparkSession = SparkSession.builder()<br>      .appName(&quot;Unit Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br>    val data = Seq(<br>     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;name1&quot;, &quot;status1&quot;, &quot;timeDiff1&quot;, &quot;active1&quot;, &quot;2021-01-01&quot;, &quot;2021-12-31&quot;, &quot;insertedBy1&quot;, &quot;2021-01-01&quot;, &quot;action1&quot;, &quot;owner1&quot;)<br>    )<br><br>    val schema = StructType(List(<br>     StructField(&quot;sourceString&quot;, StringType, false),<br>     StructField(&quot;PROP_NUM&quot;, IntegerType, false),<br>     StructField(&quot;PROP_CODE&quot;, StringType, false),<br>     StructField(&quot;UNIT&quot;, StringType, false),<br>     StructField(&quot;HGA_UNIT&quot;, StringType, false),<br>     StructField(&quot;PROP_NAME&quot;, StringType, false),<br>     StructField(&quot;PHYS_CAP&quot;, StringType, false),<br>     StructField(&quot;PRODUCT_INDEX&quot;, StringType, false),<br>     StructField(&quot;PRODUCT_CODE&quot;, StringType, false),<br>     StructField(&quot;REGION&quot;, StringType, false),<br>     StructField(&quot;GEO_REGION&quot;, StringType, false),<br>     StructField(&quot;HOTEL_TYPE&quot;, StringType, false),<br>     StructField(&quot;BILLING_TYPE&quot;, StringType, false),<br>     StructField(&quot;MGT_TYPE&quot;, StringType, false),<br>     StructField(&quot;NETWORK&quot;, StringType, false),<br>     StructField(&quot;PMS_TYPE&quot;, StringType, false),<br>     StructField(&quot;INTRFACE&quot;, StringType, false),<br>     StructField(&quot;ADDR1&quot;, StringType, false),<br>     StructField(&quot;ADDR2&quot;, StringType, false),<br>     StructField(&quot;CITY&quot;, StringType, false),<br>     StructField(&quot;STATE&quot;, StringType, false),<br>     StructField(&quot;ZIP_CODE&quot;, StringType, false),<br>     StructField(&quot;COUNTRY_CODE&quot;, StringType, false),<br>     StructField(&quot;CURRENCY_CODE&quot;, StringType, false),<br>     StructField(&quot;OPEN_DATE&quot;, StringType, false),<br>     StructField(&quot;COMP_FLAG&quot;, StringType, false),<br>     StructField(&quot;COMP_DT&quot;, StringType, false),<br>     StructField(&quot;ADD_DT&quot;, StringType, false),<br>     StructField(&quot;TIME_DIFF&quot;, StringType, false),<br>     StructField(&quot;DOLLARS&quot;, StringType, false),<br>     StructField(&quot;YMS_USER&quot;, StringType, false),<br>     StructField(&quot;ALT_ROUTE&quot;, StringType, false),<br>     StructField(&quot;ALT_ROUTE_FLAG&quot;, StringType, false),<br>     StructField(&quot;DBDB&quot;, StringType, false),<br>     StructField(&quot;KING&quot;, StringType, false),<br>     StructField(&quot;DOM_FRGN_PROP&quot;, StringType, false),<br>     StructField(&quot;INV_TYPE_YYPP&quot;, StringType, false),<br>     StructField(&quot;INV_TYPE&quot;, StringType, false),<br>     StructField(&quot;DIV&quot;, StringType, false),<br>     StructField(&quot;SHOPPE&quot;, StringType, false),<br>     StructField(&quot;ACCT_NUM&quot;, StringType, false),<br>     StructField(&quot;GROUP_ACCT&quot;, StringType, false),<br>     StructField(&quot;AMB_CODE&quot;, StringType, false),<br>     StructField(&quot;CORP_NUM&quot;, StringType, false),<br>     StructField(&quot;PHONE&quot;, StringType, false),<br>     StructField(&quot;FAX&quot;, StringType, false),<br>     StructField(&quot;AVAIL_CAP&quot;, StringType, false),<br>     StructField(&quot;SMSA&quot;, StringType, false),<br>     StructField(&quot;TRADING_AREA&quot;, StringType, false),<br>     StructField(&quot;LAT_DEGREES&quot;, StringType, false),<br>     StructField(&quot;LAT_MINUTES&quot;, StringType, false),<br>     StructField(&quot;LAT_SECONDS&quot;, StringType, false),<br>     StructField(&quot;LAT_N_OR_S&quot;, StringType, false),<br>     StructField(&quot;LON_DEGREES&quot;, StringType, false),<br>     StructField(&quot;LON_MINUTES&quot;, StringType, false),<br>     StructField(&quot;LON_SECONDS&quot;, StringType, false),<br>     StructField(&quot;LON_E_OR_W&quot;, StringType, false),<br>     StructField(&quot;X_PL_DIST_CD&quot;, StringType, false),<br>     StructField(&quot;REWARDS_CATEGORY&quot;, StringType, false),<br>     StructField(&quot;PROVINCE_CD&quot;, StringType, false),<br>     StructField(&quot;STATUS&quot;, StringType, false),<br>     StructField(&quot;STATUS_DT&quot;, StringType, false),<br>     StructField(&quot;TIME_OF_RECEIPT&quot;, StringType, false),<br>     StructField(&quot;ACTIVE_IND&quot;, StringType, false),<br>     StructField(&quot;EFF_START_DT&quot;, StringType, false),<br>     StructField(&quot;EFF_END_DT&quot;, StringType, false),<br>     StructField(&quot;LAST_UPD_BY&quot;, StringType, false),<br>     StructField(&quot;LAST_UPD_TS&quot;, StringType, false),<br>     StructField(&quot;ACTION_IND&quot;, StringType, false),<br>     StructField(&quot;OWNER&quot;, StringType, false)<br>    ))<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br>    )<br><br>    val resultDF = getFinalDataFrame(inputDF)<br><br>    // Basic assertions<br>    assert(resultDF != null, &quot;Resulting DataFrame should not be null&quot;)<br>//    assert(resultDF.columns.contains(ColumnPrefix + &quot;PCPT_SOURCE_STRING&quot;), s&quot;Resulting DataFrame should contain the column ${ColumnPrefix}PCPT_SOURCE_STRING&quot;)<br>//    assert(resultDF.columns.contains(ColumnPrefix + &quot;PCPT_PROPERTY_NUM&quot;), s&quot;Resulting DataFrame should contain the column ${ColumnPrefix}PCPT_PROPERTY_NUM&quot;)<br>//    assert(resultDF.columns.contains(JsonData), s&quot;Resulting DataFrame should contain the column $JsonData&quot;)<br>   }<br><br>ARRANGE THE DATA ACCORDINGLY TO THE SCHEMA AND IF YOU SEE ANY ISSUE IN SCHEMA LET ME KNOW<br><br><br><br><br><br><br><br>getting this error<br>GETTING THIS ISSUE C:\Users\ajune750\product-to-catalog\ProductCatalog-to-PropertyLookupETL\ProductCatalogToProperty\src\test\scala\com\marriott\mgpapp\spark\productcatalog\property\helper\ProductCatalogToPropertyJobHelperTest.scala:1013:6<br>too many elements for tuple: 70, allowed: 22<br>     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br><br>     solution:<br><br>     The error you&#39;re encountering is because Scala tuples have a maximum size limit of 22 elements. Since your data has 70 elements, you need to use a different approach to create the DataFrame.<br><br>You can use a Row object instead of a tuple to represent each row of data. Here&#39;s how you can adjust your test case:<br><br>Use Row objects to represent the data.<br>Create the DataFrame from the list of Row objects and the schema.<br>Here&#39;s the modified test case:<br><br>     //    val data = Seq(<br>//     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>//    )<br>val data = Seq(<br> Row(&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>)<br><br><br><br>C:\Users\ajune750\product-to-catalog\ProductCatalog-to-PropertyLookupETL\ProductCatalogToProperty\src\test\scala\com\marriott\mgpapp\spark\productcatalog\property\helper\ProductCatalogToPropertyJobHelperTest.scala:1092:50<br>type mismatch;<br> found   : Product =&gt; org.apache.spark.sql.Row<br> required: org.apache.spark.sql.Row =&gt; ?<br>     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br><br><br><br><br>The error you&#39;re encountering is because Scala tuples have a maximum size limit of 22 elements. Since your data has 70 elements, you need to use a different approach to create the DataFrame.<br><br>You can use a Row object instead of a tuple to represent each row of data. Here&#39;s how you can adjust your test case:<br><br>Use Row objects to represent the data.<br>Create the DataFrame from the list of Row objects and the schema.<br>Here&#39;s the modified test case:<br><br><br><br>=========================================================================================================================================================================<br><br><br><br><br><br><br><br><br><br><br>builder object ki zarrorrat kuon hain<br>hum kya mock kar skate hai<br>kya nahi <br><br>class ko nahi mock kar skate <br><br><br>individual ab connection ko mock kar sakte hai<br><br><br>function ke ander ek aur object bana ho connection ka usko mock nahi kar sakte <br><br><br>uske liye 2 solution hai<br><br><br>ya to builderutil me object bana do<br><br><br>ya jaise collation me kiya tha usko pass kara do parameter me <br><br><br><br><br>builder util ka example rateplan se<br><br>and <br><br><br><br>balki doo example mil jayenge<br><br>ek builder util wala bhi and paramter me pass karane wala bhi collation me<br><br><br>===================================================================================================================<br><br><br><br>foreachbatch not mockable<br><br>how to do it what is the method lalit used<br><br>singelton builder class me <br><br>kuonki<br><br>bar baar call karte hai<br><br>kuonki wo object hota hai<br>aur hum class ko nahi kar sakte call aur mock<br><br>singlteon ka ek hi instance hota hai<br><br>aur uska object crreste naho hota<br><br><br><br>=========================================================================================================<br>for converting a dataframe sparksession is must<br><br>   Create a SparkSession for testing<br>   val spark: SparkSession = SparkSession.builder()<br>     .appName(&quot;UnitTest&quot;)<br>     .master(&quot;local[*]&quot;)<br>     .getOrCreate()<br><br>===================================================================<br><br>         import spark.implicits._<br><br>    // Sample input data<br>    val inputData = Seq(<br>     (&quot;value1&quot;, &quot;payload1&quot;)<br>   , (&quot;value2&quot;, &quot;payload2&quot;)<br>    ).<br>     // toDF(&quot;someColumn&quot;)<br>      toDF(&quot;someColumn&quot;, &quot;payLoad&quot;)<br><br><br>      ========================================================================<br><br><br>how to make  dataframe <br>and what is tuple exceeds 22 then we nee to covert a row <br><br>parallelize also changes<br><br>       val spark: SparkSession = SparkSession.builder()<br>      .appName(&quot;Unit Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br><br>    val data = Seq(<br>     Row(&quot;event1&quot;),<br>     Row(&quot;ea oent2&quot;)<br>    )<br><br>    val schema = StructType(List(<br>     StructField(Value, StringType, false)<br>    ))<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data),<br>     schema<br>    )<br><br>    val resultDF = transformNonConvertedEntitiesToIrrecoverableDLTMessage(inputDF)<br>    assert(resultDF != null, &quot;Resulting DataFrame should not be null&quot;)<br>   }<br><br>   differentiate<br><br>   //    val data = Seq(<br>//     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>//    )<br>val data = Seq(<br> Row(&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;,&quot;mktg_HOTEL_TYP1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>)<br><br>then this as well<br><br>//    val inputDF = spark.createDataFrame(<br>//     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br>//    )<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data), schema<br>    )<br><br><br><br><br><br><br><br><br><br>    ===========================================================================================================<br><br><br>    sqlcontext.reader for query to read<br><br>    also map(ke ander jo query hoti hai wo daalni hoti hai agar error aye db map me to) key is missing<br><br>     test(&quot;getPreviousDaySnapshot should fetch snapshot data from PostgreSQL DB&quot;) {<br>    implicit val sparkSession: SparkSession = mock[SparkSession]<br>    val sqlContext = mock[org.apache.spark.sql.SQLContext]<br>    //newly added  reader for null pointer exception<br>    val reader = mock[org.apache.spark.sql.DataFrameReader]<br>    val dbDetails = DbDetails(<br>     jdbcurl = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivername = &quot;org.postgresql.Driver&quot;,<br>     username = &quot;testuser&quot;,<br>     password = &quot;testpass&quot;,<br>     poolsize = &quot;5&quot;,<br>     idletimeout = 1000L,<br>     querytimeout = scala.concurrent.duration.Duration.Inf,<br>     jdbcurlro = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivernamero = &quot;org.postgresql.Driver&quot;,<br>     usernamero = &quot;testuserro&quot;,<br>     passwordro = &quot;testpassro&quot;,<br>     poolsizero = &quot;5&quot;,<br>     idletimeoutro = 1000L,<br>     querytimeoutro = scala.concurrent.duration.Duration.Inf,<br>     jdbcurlwo = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivernamewo = &quot;org.postgresql.Driver&quot;,<br>     usernamewo = &quot;testuserwo&quot;,<br>     passwordwo = &quot;testpasswo&quot;,<br>     poolsizewo = &quot;5&quot;,<br>     idletimeoutwo = 1000L,<br>     querytimeoutwo = scala.concurrent.duration.Duration.Inf<br>    )<br>    val dbMap = Map(ProductCatalogProperty -&gt; &quot;product_catalog&quot;)<br>    val schemaName = &quot;public&quot;<br><br><br>    // Mocking SparkSession behavior<br>    when(sparkSession.sqlContext).thenReturn(sqlContext)<br>    when(sqlContext.read).thenReturn(reader)<br><br>    // Mocking DataFrame<br>    val postGresDataDB: DataFrame = mock[DataFrame]<br>    val postGresDataDF: DataFrame = mock[DataFrame]<br>    val activePostGresRecords: DataFrame = mock[DataFrame]<br><br>//    when(sqlContext.read.jdbc(anyString(), anyString(), any[java.util.Properties])).thenReturn(postGresDataDB)<br>    when(reader.jdbc(anyString(), anyString(), any[java.util.Properties])).thenReturn(postGresDataDB)<br>//    java.lang.NullPointerException was thrown.<br>    when(postGresDataDB.columns).thenReturn(Array(&quot;pcpt_active_ind&quot;, &quot;other_column&quot;))<br>   // when(postGresDataDB.toDF(anyVararg[String])).thenReturn(postGresDataDF)<br>   // not found: value anyVararg<br>   // when(postGresDataDF.filter(any())).thenReturn(activePostGresRecords)<br>//    getting this ambiguous reference to overloaded definition,<br>//    both method filter in class Dataset of type (func: org.apache.spark.sql.Row =&gt; Boolean)org.apache.spark.sql.Dataset[org.apache.spark.sql.Row]<br>    when(postGresDataDB.toDF(any[Array[String]]: _*)).thenReturn(postGresDataDF)<br>    when(postGresDataDF.filter(any[org.apache.spark.sql.Column])).thenReturn(activePostGresRecords)<br>    val result = getPreviousDaySnapshot(dbDetails, dbMap, schemaName)<br>    assert(result != activePostGresRecords)<br>  }<br><br><br><br><br><br><br><br><br><br><br><br><br><br>=====================================================================================================================================================<br><br><br><br>sneha ne jo bataya tha kisi streaming job ko batch me convert karne ka tarika copy me note hai<br><br>notepad pe bhi milega<br><br>kanta ci_sgds folder me psinventory ke liye bhi hai<br><br><br><br>===================================================================================================================<br><br><br>notebbok me bahut kuch likha hai<br><br>and oozie scripts<br><br><br>====================================================================================================================<br><br>emr, spark submit, core nodes, cleaning the emr<br><br>=========================================================================================================================================================<br><br>100000 learnings of emr<br>===============================================================================================<br><br>all the tickets<br><br><br>==============================================================================<br><br>mapgroup<br><br>================================================<br><br>rate plan how to take the last variable in try and catch<br><br> try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while interacting with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          singleToRecoverable(recoverableDf)<br>          //sparkSession.emptyDataFrame<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>    lastDate<br><br><br>    ====================================================================================<br><br><br>    import from classs as well as object<br><br>    package object hai directly import ho jayega<br><br><br>    class hai to pehle uska object banao tabhi import kar sakte ho isliye testconstant ki class nahi import ho rhi thi<br>     val dbdetail2 = new TestConstants<br>    val result = getPreviousDaySnapshot(dbdetail2, dbMap, schemaName)<br><br>    assert(result == emptyDF)<br>   }<br><br><br><br><br>   =======================================================================================<br><br><br>   fetch fetch:  sari branches sync me ho jayegi<br><br>git checkout -b feture-mgp/47628<br><br>local and remote<br><br>teams chat also<br><br><br>git clone  https://git.marriott.com/RMS/ReservationETL.git development-mgp<br><br><br>====================================================<br><br>INTELLIJ MEMORY <br><br>2GB SE 4GB KARI HELP ME JAAKE<br> SO THAT JOB TAKES LESS TIME<br><br> ================================================================<br><br> test case outstanding work <br><br> utkarsaha ne isko upar method me hi daal diya hai taaki neeche wo mock karnw hi na pade properties ko jisse eerror aa raha hai<br><br><br> same tet case me bhi value pass kar di hai<br><br><br><br>  def produceOutputMessage(dataFrame: DataFrame, destTopic: String = KafkaProperties.destinationTopic): Unit = {<br>    try {<br>      if (dataFrame.isEmpty) {<br>        log.info(&quot;Received Empty DataFrame, not inserting any records to Destination Topic | produceOutputMessage()&quot;, customMeta)<br>      } else {<br>        prepareBatchProducer(dataFrame, destTopic, Some(classOf[EEHData]), Some(&quot;source_schema.avsc&quot;))<br>        log.info(&quot;Successfully inserted records to Destination Topic | produceOutputMessage()&quot;, customMeta)<br>      }<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;produceOutputMessage() :: Exception while producing records to Kafka Topic :: Exception:&quot; + exception.printStackTrace(), customMeta)<br>        new KafkaException(exception, errorMeta).handle()<br>    }<br>  }<br><br><br><br>  test case :<br><br>  produceOutputMessage(dataFrame, &quot;destination-test-topic&quot;)<br><br><br>   Caused by: java.io.FileNotFoundException: Properties file cannot be loaded<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelperImpl.init(PropertiesHelper.scala:32)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.init(PropertiesHelper.scala:85)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.&lt;init&gt;(PropertiesHelper.scala:84)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.&lt;clinit&gt;(PropertiesHelper.scala)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties.initKafkaConfig$default$1(KafkaProperties.scala:170)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties$.&lt;init&gt;(KafkaProperties.scala:288)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties$.&lt;clinit&gt;(KafkaProperties.scala)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.getStreamInstance(KafkaCircuitBreaker.scala:28)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.&lt;init&gt;(KafkaCircuitBreaker.scala:12)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.<br><br><br>----------------------------------------------------------------------------------<br><br>preparebatchproducer<br><br><br>/    val mockProducerHelper = mock[ProducerHelper[EEHData]]<br>//    when(mockProducerHelper.produce()).thenReturn(None)<br>//    when(<br>//     prepareBatchProducer(<br>//      ArgumentMatchers.any(classOf[DataFrame]),<br>//      ArgumentMatchers.any(classOf[String]),<br>//      ArgumentMatchers.eq(Option(classOf[EEHData])),<br>//      ArgumentMatchers.any()<br>//<br>//     )(ArgumentMatchers.any())<br>//    ).<br>////      thenReturn(mockProducerHelper)<br>//      thenReturn(None)<br><br>---------------------------------------------------------------------------------------------------------------------------<br><br><br>THIS IS RecoverableDf:Dataset[MergedRatePlan],<br><br>  case class MergedRatePlan(<br>                           ratePlan: RatePlan,<br>                           sourceString: String<br>                         ) HOW TO PRINT THE sourceString: Strin IN SCALA USING RecoverableDf <br>                         TELL ME ALL THE APPROACHES<br><br>                         ANS<br><br><br><br><br>To print the sourceString field from the MergedRatePlan case class in Scala using a DataFrame (RecoverableDf), you can use several approaches depending on your specific needs. Here are some of the common methods:<br><br>1. Using the show Method<br>If you want to display the DataFrame and include the sourceString column, you can use the show method:<br><br>RecoverableDf.show()<br>This will print the entire DataFrame to the console, including the sourceString column.<br><br>2. Selecting and Showing the sourceString Column<br>If you only want to print the sourceString column:<br><br>RecoverableDf.select(&quot;sourceString&quot;).show()<br>3. Collecting the Data and Iterating<br>You can collect the data to the driver and then iterate over the rows to print each sourceString:<br><br>RecoverableDf.select(&quot;sourceString&quot;).collect().foreach(row =&gt; println(row.getString(0)))<br>4. Using Dataset API<br>If RecoverableDf is already a Dataset of MergedRatePlan, you can use the Dataset API to extract and print the sourceString:<br><br>RecoverableDf.map(_.sourceString).foreach(println)<br>5. Using the rdd Method<br>You can convert the DataFrame to an RDD and then use RDD operations to print the sourceString:<br><br>RecoverableDf.select(&quot;sourceString&quot;).rdd.map(row =&gt; row.getString(0)).foreach(println)<br>6. Using SQL Queries<br>If you want to use SQL queries, you can register the DataFrame as a temporary view and run a SQL query:<br><br>RecoverableDf.createOrReplaceTempView(&quot;merged_rate_plan_view&quot;)<br>val sqlDF = spark.sql(&quot;SELECT sourceString FROM merged_rate_plan_view&quot;)<br>sqlDF.show()<br>7. Using withColumn Method<br>If you want to create a new DataFrame with only the sourceString column and then print it:<br><br>val sourceStringDf = RecoverableDf.withColumn(&quot;sourceString&quot;, col(&quot;sourceString&quot;))<br>sourceStringDf.show()<br>8. Using filter and select<br>If you have specific filtering criteria and then want to print the sourceString:<br><br>RecoverableDf.filter($&quot;sourceString&quot;.isNotNull).select(&quot;sourceString&quot;).show()<br>Remember to import the necessary Spark SQL and DataFrame functions:<br><br>import org.apache.spark.sql.SparkSession<br>import org.apache.spark.sql.functions._<br>Example Setup<br>Here is a small example setup to illustrate the usage:<br><br>import org.apache.spark.sql.{SparkSession, Dataset}<br><br>case class RatePlan(planName: String)<br>case class MergedRatePlan(ratePlan: RatePlan, sourceString: String)<br><br>val spark = SparkSession.builder.appName(&quot;Example&quot;).getOrCreate()<br>import spark.implicits._<br><br>// Sample data<br>val data = Seq(<br>  MergedRatePlan(RatePlan(&quot;PlanA&quot;), &quot;SourceA&quot;),<br>  MergedRatePlan(RatePlan(&quot;PlanB&quot;), &quot;SourceB&quot;)<br>)<br><br>// Creating DataFrame<br>val RecoverableDf: Dataset[MergedRatePlan] = data.toDS()<br><br>// Printing sourceString using different approaches<br>RecoverableDf.show()<br>RecoverableDf.select(&quot;sourceString&quot;).show()<br>RecoverableDf.select(&quot;sourceString&quot;).collect().foreach(row =&gt; println(row.getString(0)))<br>RecoverableDf.map(_.sourceString).foreach(println)<br>RecoverableDf.select(&quot;sourceString&quot;).rdd.map(row =&gt; row.getString(0)).foreach(println)<br>RecoverableDf.createOrReplaceTempView(&quot;merged_rate_plan_view&quot;)<br>val sqlDF = spark.sql(&quot;SELECT sourceString FROM merged_rate_plan_view&quot;)<br>sqlDF.show()<br>val sourceStringDf = RecoverableDf.withColumn(&quot;sourceString&quot;, col(&quot;sourceString&quot;))<br>sourceStringDf.show()<br>RecoverableDf.filter($&quot;sourceString&quot;.isNotNull).select(&quot;sourceString&quot;).show()<br>Choose the method that best fits your use case.<br><br><br>==============================================================================================================================<br><br><br><br><br><br><br><br><br><br><br><br><br><br>full article prepared**************************************************************************************************************************************<br><br>  this is the method,<br><br>                         def singleToRecoverable(inputDS: Dataset[MergedRatePlan])(implicit sparkSession: SparkSession): Unit = {<br>    log.info(&quot;entered into recoverable topic&quot;, customMeta)<br>    try {<br>      val finalDS = inputDS<br>        .select(col(&quot;sourceString&quot;))<br>        //.coalesce(5)<br>        .select(from_json(col(&quot;sourceString&quot;)<br>          , Encoders.product[com.marriott.mmf.RatePlan].schema).as(&quot;value&quot;))<br><br><br>this is line 120 (RatePlanIngestionHelper.scala:120) where its showing the below error <br><br>                               class:c.m.m.s.r.i.j.RatePlanIngestionJob - Exception while having a connect with DB :: Exception: () - <br>2024-09-16 15:54:42.083 [Executor task launch worker for task 163.0 in stage 15.0 (TID 1011)] INFO <br>                                class:c.m.m.s.r.i.j.RatePlanIngestionJob - entered into recoverable topic - org.apache.spark.SparkException:  Dataset transformations and actions can only be invoked by the driver, not inside of other Dataset transformations; for example, dataset1.map(x =&gt; dataset2.values.count() * x) is invalid because the values transformation and count action cannot be performed inside of the dataset1.map transformation. For more information, see SPARK-28702.       <br>  at org.apache.spark.sql.errors.QueryExecutionErrors$.transformationsAndActionsNotInvokedByDriverError(QueryExecutionErrors.scala:1967)<br>  at org.apache.spark.sql.Dataset.sparkSession$lzycompute(Dataset.scala:198)<br>  at org.apache.spark.sql.Dataset.sparkSession(Dataset.scala:196)<br>  at org.apache.spark.sql.Dataset.withPlan(Dataset.scala:3887)<br>  at org.apache.spark.sql.Dataset.select(Dataset.scala:1519)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.singleToRecoverable(RatePlanIngestionHelper.scala:120)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.singleToRecoverable$(RatePlanIngestionHelper.scala:116)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.jobs.RatePlanIngestionJob.singleToRecoverable(RatePlanIngestionJob.scala:20)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.$anonfun$persistDB$1(RatePlanIngestionHelper.scala:508)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.$anonfun$persistDB$1$adapted(RatePlanIngestionHelper.scala:499)<br>  at scala.collection.Iterator.foreach(Iterator.scala:943)<br><br><br><br>  this is the method from where singleToRecoverable  is invoked<br>   def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf:Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit  sparkSession: SparkSession,jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]) = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;,<br>      customMeta)<br>    ratePlanBookingLimitDS.foreach { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br><br>        dbSinkObject.process(row)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          singleToRecoverable(recoverableDf)<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>    lastDateDF.foreach{dates=&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>        //delete stay_date &gt; last_date<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>  }<br>}<br><br><br><br> def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    val failedRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(row) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>    if (failedRows.nonEmpty) {<br>      singleToRecoverable(recoverableDf)<br>    }<br><br>    lastDateDF.foreach { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>  }<br>}<br><br><br>we need to pass the same singleToRecoverable(recoverableDf) in the catch block if we find the exception for this  lastDateDF.foreach { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      } finally {<br>        dbSinkObject.close(null)<br>      }, now give me the modified persistDB function according to that. <br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>      the final solution:<br><br><br><br><br><br><br><br>      ===========approcah 1=============================================================================================================<br><br><br><br>     To resolve this issue, you need to ensure that singleToRecoverable is invoked from the driver rather than within the foreach action. One approach is to collect the data to the driver and then process it, or handle the exceptions differently.<br><br>Approach 1: Collect to Driver and Process<br>This approach involves collecting the Dataset to the driver and then processing each element. This ensures that singleToRecoverable is invoked on the driver side.<br><br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  val ratePlanBookingLimitArray = ratePlanBookingLimitDS.collect() // Collect to driver<br>  ratePlanBookingLimitArray.foreach { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        singleToRecoverable(recoverableDf) // This is now safe<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  val lastDateArray = lastDateDF.collect() // Collect to driver<br>  lastDateArray.foreach { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br>}<br><br><br><br><br><br><br><br><br><br><br>==========================approach 2=======================================================================<br><br>Approach 2: Handle Exceptions Differently<br>Another approach is to handle the exceptions such that singleToRecoverable is not invoked directly<br> within the foreach action. Instead, you can collect the failed rows and process them separately on the driver.<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>====================question----------------------------------------------------------------------<br><br>in the above case why we have chaned foreach to flatmap they both are transformations and most probably same error will occur in bot the cases<br><br>Copy<br><br><br>answer--------------------------------------------------------------------------------------------------------------<br><br>You are correct that both foreach and flatMap are transformations; however, the crucial difference lies in how they are used and where the actions are triggered. The objective is to ensure that Spark operations (transformations and actions) are not nested within actions. <br><br>In the previous solution, flatMap was used to accumulate failed rows, and the collection of these rows happens as a separate action (collect), which is performed at the driver level. This ensures that singleToRecoverable is invoked on the driver.<br><br>However, if flatMap doesn&#39;t seem to resolve the issue and the error persists, we need to ensure that no Spark transformations or actions are nested within actions. <br><br>Refactoring the Code:<br>Let&#39;s ensure that all Spark operations are executed at the driver level by collecting the entire Dataset to the driver before processing. Here&#39;s a revised approach:<br><br>Collect the ratePlanBookingLimitDS and lastDateDF to the driver.<br>Process each row on the driver.<br>Handle exceptions and invoke singleToRecoverable appropriately.<br><br><br><br>======================================================================<br><br><br>final<br><br><br>question==============================================================================<br><br>just confirming this wont impact the succesfull scenario where it does not catch  the error  as we have changed from foreach to flatmap<br><br>Copy================answer========================================================<br><br>Switching from foreach to flatMap in the context provided earlier should not impact the successful scenario where no errors are encountered. However, there is a subtle difference in how these two operations work:<br><br>foreach: Executes a function for each element in the Dataset. This is an action and triggers the computation.<br>flatMap: Applies a function to each element in the Dataset and flattens the results. This is a transformation and is lazy until an action is called on the resulting Dataset.<br><br><br>Ensuring No Impact on Successful Scenario:<br>The main goal is to ensure that we handle the data processing correctly and only invoke singleToRecoverable when an error occurs. The refactored approach should work seamlessly for both successful and error scenarios because:<br><br>Data Processing: Data is processed using foreach after collecting it to the driver, ensuring that each row is handled individually.<br>Error Handling: Errors are captured and used to set a flag, which controls whether singleToRecoverable is invoked.<br>Control Flow: The control flow remains the same for successful scenarios—each row is processed, and singleToRecoverable is only called if an error occurs.<br><br><br>Final Code with Explanation:<br>Let&#39;s revalidate the final code to ensure it correctly handles both successful and error scenarios.<br><br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  // Collect the datasets to the driver<br>  val ratePlanBookingLimitArray = ratePlanBookingLimitDS.collect() <br>  val lastDateArray = lastDateDF.collect()<br><br>  var hasExceptionOccurred = false<br><br>  // Process ratePlanBookingLimitArray on the driver<br>  ratePlanBookingLimitArray.foreach { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  // Process lastDateArray on the driver<br>  lastDateArray.foreach { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  // If any exception occurred, invoke singleToRecoverable<br>  if (hasExceptionOccurred) {<br>    singleToRecoverable(recoverableDf)<br>  }<br>}<br>Explanation:<br><br>Collecting Data: <br>ratePlanBookingLimitDS.collect() and lastDateDF.collect() ensure all data is available on the driver, avoiding any distributed processing issues.<br><br><br>Processing Data: <br>ratePlanBookingLimitArray.foreach { row =&gt; ... } and lastDateArray.foreach { dates =&gt; ... } process each element individually. This is the same logic that was previously inside the foreach on the Dataset but now executes on the driver.<br><br><br>Error Handling: <br>Errors are captured, and a flag (hasExceptionOccurred) is set to true if any error occurs.<br><br><br>Control Flow: <br>If no errors occur, hasExceptionOccurred remains false, and singleToRecoverable is not invoked.<br>If any error occurs, hasExceptionOccurred is set to true, and singleToRecoverable(recoverableDf) is invoked after processing all rows.<br><br><br>Conclusion:<br>This approach ensures that the successful scenario where no errors are encountered will not be impacted. The data processing logic remains the same, and singleToRecoverable is only invoked when necessary, based on the error flag. This maintains the integrity of the original logic while adhering to Spark&#39;s execution model.<br><br><br>Copy<br><br><br><br><br>----------------------------------<br><br>this is the method:<br><br><br>  def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(row) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>    // Process lastDateDF and collect failed rows<br>    val failedDateRows = lastDateDF.flatMap { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(dates) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimitDeletion]).collect() // Collect the failed rows to the driver<br><br>    // Invoke singleToRecoverable if there are any failed rows<br>    if (failedRatePlanRows.nonEmpty || failedDateRows.nonEmpty) {<br>      singleToRecoverable(recoverableDf)<br>    }<br>  }<br><br><br><br><br>  ============================changed to the new one<br><br><br><br>  COLLECT KIYA ...DATA DRIVER PE AA GYA<br><br>  DRIVER RETURN KAR DIYA<br><br>  256MB PE FAIL KAR JAYEGI<br><br><br>  COLLECT KARTE HAI HUM ...<br><br><br><br>  REPARTITION KARTE 5 <br><br>  PER  EXECUTIONER PARTITIONING KE LIYE<br><br>  100 EVENT HAI EXECUTER , 5 SE DIVIDE KARKE 20 HOGAYA<br>  <br>  TO WO EK <br><br><br><br><br>*******************************************************************************************************************************************************************<br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  // Process ratePlanBookingLimitDS and collect failed rows<br>  val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        Some(row) // Collect the failed row<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>  // Process lastDateDF and collect failed rows<br>  val failedDateRows = lastDateDF.flatMap { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>      None<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        Some(dates) // Collect the failed row<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }(Encoders.product[RatePlanBookingLimitDeletion]).collect() // Collect the failed rows to the driver<br><br>  // Invoke singleToRecoverable if there are any failed rows<br>  if (failedRatePlanRows.nonEmpty || failedDateRows.nonEmpty) {<br>    singleToRecoverable(recoverableDf)<br>  }<br>}<br><br>for the above solution you provided please put foreach and dont replace it with flatmap, also dont use collect as it in not optimised<br>you can do one thing you can start the try block  from the start of the method and hencforth adjust catch block as well<br><br><br>like for example<br><br>dont start from here  try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    }  rather you can start from here<br><br><br> try {<br>    val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>   <br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    }<br><br><br><br><br>    =====================================final solution===============================================================================<br><br>      def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    var hasExceptionOccurred = false // Flag to track if any exception occurs<br><br>    try {<br>      // Process ratePlanBookingLimitDS using foreach<br>      ratePlanBookingLimitDS.foreach { row =&gt;<br>        val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>        try {<br>          dbSinkObject.open(1.toLong, 2)<br>          dbSinkObject.process(row)<br>        } catch {<br>          case exception: Exception =&gt;<br>            log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>            hasExceptionOccurred = true<br>        } finally {<br>          dbSinkObject.close(null)<br>        }<br>      }<br><br>      // Process lastDateDF using foreach<br>      lastDateDF.foreach { dates =&gt;<br>        val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>        try {<br>          dbSinkObject.open(1.toLong, 2)<br>          dbSinkObject.deleteRecords(dates)<br>        } catch {<br>          case exception: Exception =&gt;<br>            log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>            hasExceptionOccurred = true<br>        } finally {<br>          dbSinkObject.close(null)<br>        }<br>      }<br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception encountered during persistDB execution: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    }<br><br>    // If any exception occurred, invoke singleToRecoverable<br>    if (hasExceptionOccurred) {<br>      singleToRecoverable(recoverableDf)<br>    }<br>  }<br><br>  <br><br><br>  ***********************************************************************************************************************************************************************************************************88reservation  ingestion how to give sparksession and df empty in return<br><br> catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>            jobName -&gt; jobConfigDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records&quot;&quot;&quot;.stripMargin<br>            }<br>          )<br>        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br><br>    }<br><br><br><br>    --------------------==============================================================================================================================================<br><br><br>    merge conflicts<br><br><br>    pehle to uncommit kiya <br>    usse pehle stash kana zaruri hai<br><br>    uncommit ho jata hai<br>    sourcetree se=============================================<br><br><br>    ab merge conflict aaye<br><br><br>    pehle to apni branch me jao<br><br>    fir git pull dev-mgp<br><br><br>    ab issme intellij pe aa gaye merge conflict unko resolve karo<br><br><br>    ==============================================<br><br><br>    git pull me sari dusre folder bhi stage ho jate hai<br><br>    isliye git rebase karna chahiye wo directly merge nahi karta hua<br><br><br><br>    simple use sourcetree<br><br>    =================================================================<br><br>    <br><br><br><br><br><br><br>     import sparkSession.implicits._<br><br>     without this was getting the error <br><br>         val payloadEvent = filteredReservationDS.map(x =&gt; x.reservationPayload.payLoad).toDF<br><br>error : no implicit found for paramter  encoder<br><br><br>===================================================================================================================<br><br><br>two methods different catch exceptions<br><br><br>ist one:<br><br> def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb: &quot; + ex.printStackTrace())<br>        List.empty[ReservationCollationEvent]<br>    }<br>  }<br><br><br><br>  2nd method==============================================================================<br>   def fetchPMSfromRateplan(<br>                              reservationRecordsList: List[ReservationCollationEvent]<br>                            )(implicit jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails):<br>    List[ReservationCollationEvent] = {<br><br>      val updatedList = reservationRecordsList. map { row =&gt;<br>        try {<br>          val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br><br>            val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>              if (ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;).isEmpty) {<br><br>                val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>                val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) // -- 2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>                val mongodbClient = getMongoClient(conf, DocumentDBInput)<br><br>                val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>                val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>                val key = &quot;RATEDFENDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + row.reservationPayload.HotelReservation.get.ratePlans.head.code + &quot;_&quot;<br>                val regexPattern = s&quot;&quot;&quot;^$key.*_$$&quot;&quot;&quot;.r.pattern<br>                val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>                val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>                val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>                val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>                val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>                rateplanupdated<br>              } else {<br><br>                ratePlan<br>              }<br>            }<br>            rec.copy(ratePlans = updatedRatePlans)<br>          }<br>          row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>        } catch {<br>          case exception: Exception =&gt;<br>            new ExceptionHandler(exception).handle(<br>              Map(<br>                jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>                jobName -&gt; jobConfigurationDetails.jobname,<br>                customErrorMessage -&gt; {<br>                  s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from<br>                     |${jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;)} for reservation ID ${row.reservationInfo}&quot;&quot;&quot;.stripMargin<br>                }<br>              )<br>            )<br>            ReservationCollationEvent(row.reservationInfo, row.reservationPayload, isRecoverable = true)<br>        }<br>      }<br><br>      updatedList<br>    }<br><br><br>    ================================================================================================<br><br><br>    wither take a flag true or flase (rateplan recoverable)<br><br>    or try catch<br><br><br><br><br><br><br><br>    ==============================================================================================<br><br>    mappartion and mapgroup difference:<br><br><br>    this time it went to the catch block of  getLatestReservationRecord as this (Exception in getLatestReservationRecord:) is printed in the logs  <br>catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>        throw ex // rethrow the exception to be caught by reservationCollation<br>    }<br><br><br>but again it didnt go to the exception of <br><br><br>idealy if we get the exception in getLatestReservationRecord which is enclosed in thr try block of reservationCollation method it should have gone to catch block of  reservationCollation<br><br>what is causing the issue<br><br>def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    try {<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups(getLatestReservationRecord)<br>      //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.info(&quot;entering in the catch block of reservationCollation &quot;)<br>//        new ExceptionHandler(exception).handle(<br>//          Map(<br>//            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>//            jobName -&gt; jobConfigDetails.jobname,<br>//            customErrorMessage -&gt; {<br>//              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records-2&quot;&quot;&quot;.stripMargin<br>//            }<br>//          )<br>//        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        println(&quot;printing schema of  recoverableEvent&quot;)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br>Copy<br><br>To ensure that exceptions thrown in getLatestReservationRecord are caught by the reservationCollation method, several factors must be considered, especially with respect to how Spark handles exceptions in distributed computing contexts. One common issue is that Spark operations often execute in separate threads or executors, and exceptions may not propagate as expected to the driver program&#39;s try-catch blocks.<br><br>Here&#39;s a checklist and potential fixes to ensure exceptions propagate correctly:<br><br>Ensure Rethrown Exception:<br>Confirm that getLatestReservationRecord actually rethrows the exception after logging it.<br><br><br>Exception Handling in Spark Transformations:<br>When using Spark transformations (e.g., mapGroups), exceptions thrown in these transformations may not propagate back to the driver. Instead, they might be logged on the executor nodes.<br><br><br>Use mapPartitions for Better Exception Handling:<br>If you suspect the exception handling issue is related to the distributed nature of Spark, consider using mapPartitions to handle exceptions more granularly.<br><br><br>Checkpointing and Debugging:<br>Add additional logging to confirm the flow of execution and pinpoint where the exception might be getting swallowed or lost.<br>Here is a modified version of reservationCollation that includes more detailed logging and uses mapPartitions for better exception handling:<br><br>def reservationCollation(<br>    rawReservationDS: Dataset[ReservationCollationEvent],<br>    sparkSession: SparkSession<br>)(implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups((key, records) =&gt; {<br>        try {<br>          getLatestReservationRecord(Some(key), records)<br>        } catch {<br>          case ex: Exception =&gt;<br>            log.error(&quot;Exception in mapGroups while processing key: &quot; + key, ex)<br>            throw ex<br>        }<br>      })<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;)<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;)<br>    (newNOldRecordsDS, recoverableDocDBDF)<br><br>  } catch {<br>    case exception: Exception =&gt;<br>      log.error(&quot;Exception in reservationCollation: &quot;, exception)<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      log.info(&quot;Printing schema of recoverableEvent&quot;)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      log.info(&quot;Checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>Key Changes and Additions:<br><br><br>Detailed Logging:<br>Added more logging to both the try and catch blocks to track the flow and pinpoint where the exception is being handled or missed.<br><br><br>Exception Handling in mapGroups:<br>Wrapped the logic within mapGroups in a try-catch block to catch and log exceptions specific to each group key.<br><br><br>Error Logging in reservationCollation Catch Block:<br>Ensured the exception is logged with a detailed message.<br>These changes should help in diagnosing the issue and ensure that exceptions are propagated correctly to the catch block of reservationCollation.<br><br><br>Copy<br>Thank you for your feedback.<br>Icon<br>============================================================================================================================<br><br><br>in case of throw use right and left in case of exceptions:<br><br><br><br><br>It seems that the exception is being logged in the mapGroups function, but it is not propagating up to the try block in reservationCollation. This behavior is typical in Spark, where exceptions occurring in distributed operations (like those inside mapGroups) do not always propagate back to the driver in a straightforward manner.<br><br>To handle exceptions in Spark operations more effectively, you can use a combination of techniques:<br><br>Collecting Results for Error Handling:<br>One approach is to collect the results of operations to the driver, where you can handle exceptions more reliably.<br><br><br>Using Accumulators or Broadcast Variables:<br>You can use Spark&#39;s accumulators or broadcast variables to signal errors back to the driver.<br><br><br>Custom Error Handling Logic:<br>Implement custom error handling within the distributed operations and handle as much as possible locally before propagating a summary error back to the driver.<br>Here&#39;s an example of how you might rework the mapGroups function to use a custom error-handling approach:<br><br>Modified getLatestReservationRecord with Error Handling:<br>def getLatestReservationRecord(<br>    reservationKey: Option[String],<br>    reservationRecords: Iterator[ReservationCollationEvent]<br>)(implicit conf: SparkConf, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): Either[Exception, NewNOldReservationRecordsErrorEntity] = {<br><br>  log.info(&quot;Fetching the latest reservation record out of list of reservation records with same reservation key - getLatestReservationRecord()&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  val reservationRecordsList = reservationRecords.toList<br><br>  try {<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br>    Right(resRecWithCMS)<br>  } catch {<br>    case ex: Exception =&gt;<br>      log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>      Left(ex)<br>  }<br>}<br>Modified reservationCollation with Error Handling:<br>def reservationCollation(<br>    rawReservationDS: Dataset[ReservationCollationEvent],<br>    sparkSession: SparkSession<br>)(implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups((key, records) =&gt; {<br>        getLatestReservationRecord(Some(key), records) match {<br>          case Right(result) =&gt; result<br>          case Left(ex) =&gt;<br>            log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>            throw ex<br>        }<br>      })<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;)<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;)<br>    (newNOldRecordsDS, recoverableDocDBDF)<br><br>  } catch {<br>    case exception: Exception =&gt;<br>      log.error(&quot;Exception in reservationCollation: &quot;, exception)<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      log.info(&quot;Printing schema of recoverableEvent&quot;)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      log.info(&quot;Checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>Explanation:<br><br><br>Returning Either from getLatestReservationRecord:<br>This method now returns an Either type, encapsulating either a successful result (Right) or an exception (Left).<br><br><br>Handling Either in mapGroups:<br>The mapGroups function processes the Either result, logging and rethrowing exceptions when they occur.<br><br><br>Logging and Exception Handling:<br>Added detailed logging to track where exceptions occur and ensure they are logged and rethrown properly.<br>With these changes, exceptions should be captured more effectively and propagate as expected to the reservationCollation method&#39;s catch block<br><br><br>===========================================================================<br><br><br>expalin these functions groupby key  or mapgroup or anything used wirth a very simple basic example shwoing how the transformation in happeing draw the tables as well  val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity]) <br><br> val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br>ChatGPT<br>In Spark&#39;s DataFrame API, groupByKey, mapGroups, and flatMap are operations used for processing data in a distributed manner. Let&#39;s break down each step with a simple example and draw tables to illustrate the transformation.<br><br>Suppose we have a dataset of GroupBlockEntity objects where each object contains information about a group block. We want to group these entities by a specific key, perform some transformation on each group, and then flatten the resulting groups into a single dataset.<br><br>Here&#39;s a basic example:<br><br>Suppose we have the following dataset of GroupBlockEntity:<br><br>Row ID  Group ID  Data<br>1 A ...<br>2 B ...<br>1 C ...<br>3 A ...<br>2 B ...<br>We want to group these entities by the Row ID column.<br><br>groupByKey: This operation groups the data by the specified key (Row ID in our case).<br>After groupByKey operation, the data would be grouped like this:<br><br>Row ID  Group<br>1 [Entity1, Entity3]<br>2 [Entity2, Entity5]<br>3 [Entity4]<br><br><br>Reservation_key :  Dataset[(ReservationCollationEvent,String), <br>  Reservation_key   :               Dataset[(ReservationCollationEvent_1)]<br>mapGroups: This operation applies a function to each group. Let&#39;s say transformGroupBlockRecords is a function that takes a group and applies some transformation to it.<br>After applying transformGroupBlockRecords, the data might look like this:<br><br>Row ID  Transformed Group<br>1 [TransformedEntity1, TransformedEntity3]<br>2 [TransformedEntity2, TransformedEntity5]<br>3 [TransformedEntity4]<br>flatMap: This operation flattens the groups into a single dataset.<br>After flatMap, the dataset would be:<br><br>Transformed Entity<br>TransformedEntity1<br>TransformedEntity3<br>TransformedEntity2<br>TransformedEntity5<br>TransformedEntity4<br>So, combining all these operations together in Spark code, we have:<br><br>scala<br>Copy code<br>val dataGB: Dataset[GroupBlockEntity] = gbDataset<br>  .groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>  .mapGroups(transformGroupBlockRecords)<br>  .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br>This code groups the dataset by rowId, applies the transformGroupBlockRecords function to each group, and then flattens the result into a single dataset.<br><br><br><br>example of the same<br>Suppose we have a dataset containing sales data, and we want to calculate the total revenue for each product. Each row represents a sale with the following schema:<br><br>Product ID  Quantity  Price<br>101 2 10<br>102 1 20<br>101 3 15<br>103 2 25<br>102 2 18<br>We want to group this data by Product ID, calculate the total revenue for each product (which is the sum of Quantity * Price for each sale), and then flatten the result.<br><br>Here&#39;s how you can achieve this in Spark:<br><br>scala<br>Copy code<br>// Define a case class for the sales data<br>case class Sale(productId: Int, quantity: Int, price: Int)<br><br>// Example data<br>val salesData = Seq(<br>  Sale(101, 2, 10),<br>  Sale(102, 1, 20),<br>  Sale(101, 3, 15),<br>  Sale(103, 2, 25),<br>  Sale(102, 2, 18)<br>)<br><br>// Create a DataFrame from the sales data<br>val salesDF = salesData.toDF<br><br>// Import necessary Spark functions<br>import org.apache.spark.sql.functions._<br><br>// Group by &#39;productId&#39;, calculate total revenue, and flatten the result<br>val totalRevenueDF = salesDF<br>  .groupByKey(_.getAs[Int](&quot;productId&quot;))<br>  .mapGroups { case (productId, iterator) =&gt;<br>    val totalRevenue = iterator.map { row =&gt;<br>      row.getAs[Int](&quot;quantity&quot;) * row.getAs[Int](&quot;price&quot;)<br>    }.sum<br>    (productId, totalRevenue)<br>  }<br>  .flatMap { case (productId, totalRevenue) =&gt;<br>    Iterator((productId, totalRevenue))<br>  }<br>  .toDF(&quot;productId&quot;, &quot;totalRevenue&quot;)<br><br>// Show the result<br>totalRevenueDF.show()<br>Output:<br><br>diff<br>Copy code<br>+---------+-------------+<br>|productId|totalRevenue|<br>+---------+-------------+<br>|      101|          65|<br>|      103|          50|<br>|      102|          76|<br>+---------+-------------+<br>In this example:<br><br>We define a case class Sale to represent the sales data.<br>We create a DataFrame salesDF from the example data.<br>We import necessary Spark functions.<br>We group the DataFrame by &#39;productId&#39;.<br>Inside mapGroups, we calculate the total revenue for each group.<br>Finally, we flatten the result using flatMap and convert it to a DataFrame totalRevenueDF.<br><br><br><br><br>===============================================================================</pre><p>general</p><pre>how to read a csv column with headers<br><br>from pyspark.sql.functions import col<br><br># Read the CSV file with the correct options<br>df2 = spark.read.option(&quot;header&quot;, &quot;true&quot;).option(&quot;sep&quot;, &quot;|&quot;).csv(&quot;s3://triton-audience-platform-delta-lake-staging/to_transunion_for_encryption/2453_10000411_INPUT_D20250217_Part11.psv&quot;)<br><br># Display the schema to verify the column names<br>df2.printSchema()<br><br># Filter the DataFrame where &#39;Unparsed Name Format&#39; is not null<br>df_filtered = df2.filter(col(&#39;Unparsed Name Format&#39;).isNotNull())<br><br><br># Display the filtered DataFrame<br>display(df_filtered)<br><br><br><br>df3 = df2.filter(col(&#39;Mobile Advertising ID (MAID) and MAID Type Array&#39;).isNotNull())<br>display(df3)</pre><p>all questions:</p><pre>seq.to ds.todf<br><br>https://git.marriott.com/RMS/RateETL/blob/feature-mgp/RMMGP-43548/PricePlanIngestion/src/test/scala/com.marriott.mgpapp.spark.priceplan.test/helper/PricePlanIngestionHelperTest.scala<br><br>this is main commit :<br><br>https://git.marriott.com/RMS/RateETL/commit/591ecf5eab90ba5c9422dc1994450114b41554e5<br><br><br> val genTimeDataset = Seq(<br>          (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>        ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>        val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br><br><br><br><br>=============================================================================================<br><br>        medium article<br><br><br>empty datfame<br>utkarsha<br><br><br><br> val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br><br>      // Create an empty DataFrame with the same schema as recoverableDocDBDF<br>      val schema = recoverableDocDBDS.schema<br>      val emptyDataFrame = sparkSession.createDataFrame(sparkSession.sparkContext.emptyRDD[Row], schema)<br><br>      (emptyDataset, emptyDataFrame)<br><br><br>      ==================================================================================================================<br><br>      why do we use Seq explain with simple example whts is it need here<br><br>val validRow = Row(<br>      &quot;&quot;,<br>      &quot;aoppor&quot;,<br>      &quot;aquote&quot;,<br>      &quot;Create&quot;,<br>      &quot;2023-12-26T23:59:54.000Z&quot;,<br>      &quot;13295284&quot;,<br>      &quot;MEMWI&quot;,<br>      &quot;GA4&quot;,<br>      &quot;Group Contract Name&quot;,<br>      &quot;Convention&quot;,<br>      &quot;2023-12-23&quot;,<br>      &quot;2023-12-25&quot;,<br>      &quot;2023-12-20T23:59:59&quot;,<br>      &quot;2023-12-26T23:59:59&quot;,<br>      &quot;KING&quot;,<br>      null,<br>      &quot;2019-11-22&quot;,<br>      &quot;2019-12-20&quot;,<br>      &quot;C&quot;,<br>      &quot;110&quot;,<br>      &quot;USD&quot;,<br>      &quot;0&quot;,<br>      &quot;0&quot;,<br>      &quot;MEMWIGA413295284&quot;<br>    )<br>    // Define SparkSession for testing<br>    val sparkSS: SparkSession =<br>      SparkSession.builder.appName(&quot;sample&quot;).master(local).getOrCreate()<br>    val row = sparkSS.sparkContext.parallelize(Seq(validRow))<br> <br> <br> why case class data is converted to sequence first anf then dataframe , why not directly to dataframe<br> <br> <br> Converting case class data to a DataFrame in Apache Spark typically involves an intermediate step of converting the data to a sequence or collection. This intermediate step is often taken to leverage certain functionalities and ensure type safety. Here&#39;s a more detailed breakdown of why this process is commonly used:<br><br>1. Type Safety and Schema Inference<br><br>Case Class Definition: A case class in Scala allows you to define a schema with fields and types. This is useful because it provides a clear and strongly-typed schema.<br>Sequence Conversion: When you convert a collection of case class instances (like Seq[MyCaseClass]) to a DataFrame, Spark can automatically infer the schema based on the case class definition. This means that you get a DataFrame with the correct column names and types without having to manually define the schema.<br><br><br>2. Ease of Use and Readability<br><br>Intermediate Sequence: Converting to a sequence makes it easier to handle and manipulate the data before creating the DataFrame. This intermediate step allows for transformations, filtering, and other operations using Scala&#39;s rich collection API.<br>Direct Conversion: While it is possible to directly convert raw data (like RDDs) to a DataFrame, doing so usually requires additional steps for schema definition and might not be as straightforward or readable.<br><br><br>3. Performance Considerations<br><br>Optimized Execution: Spark optimizes the execution of transformations and actions. By converting to a sequence and then to a DataFrame, you allow Spark to better optimize the operations on the data, leveraging its Catalyst optimizer for efficient execution.<br><br><br>4. Flexibility<br><br>Data Manipulation: By converting to a sequence first, you have more flexibility to manipulate the data using Scala&#39;s powerful collection operations before converting it to a DataFrame. This can be particularly useful for preprocessing or cleaning the data.<br><br><br>Example Code<br>Here&#39;s an example to illustrate the process:<br><br>// Define a case class<br>case class Person(name: String, age: Int)<br><br>// Create a sequence of case class instances<br>val peopleSeq = Seq(<br>  Person(&quot;Alice&quot;, 29),<br>  Person(&quot;Bob&quot;, 31),<br>  Person(&quot;Charlie&quot;, 25)<br>)<br><br>// Convert the sequence to a DataFrame<br>import spark.implicits._<br>val peopleDF = peopleSeq.toDF()<br><br>// Show the DataFrame<br>peopleDF.show()<br>In this example:<br><br>A case class Person is defined.<br>A sequence of Person instances is created.<br>The sequence is then converted to a DataFrame using the toDF method, which automatically infers the schema from the case class.<br><br><br>Direct Conversion (Alternative)<br>While it is less common, you can directly create a DataFrame from an RDD of case class instances:<br><br>val peopleRDD = spark.sparkContext.parallelize(peopleSeq)<br>val peopleDF = spark.createDataFrame(peopleRDD)<br>peopleDF.show()<br>This method is less straightforward and might not leverage all the benefits of type safety and schema inference as effectively as the sequence-to-DataFrame approach.<br><br>Conclusion<br>The common practice of converting case class data to a sequence before creating a DataFrame is due to reasons of type safety, ease of use, performance optimization, and flexibility. While direct conversion is possible, the intermediate sequence step often provides significant advantages.<br><br><br>Copy<br>Thank you for your feedback.<br>Icon<br><br><br>==============================================collation==================================================================================<br><br><br>  def genarateEEHData(sparkSession: SparkSession): DataFrame = {<br>    import sparkSession.implicits._<br>    sparkSession.createDataset(List(TestsHelper.toString)).as(&quot;value&quot;).toDF()<br>  }<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>  ====================================tag===============================================<br>  Tag: Used to mark specific rows and identify table instances.<br>AbstractTable: Provides the framework for defining table schemas.<br>Usage: Tags are typically used internally by Slick to manage and differentiate between table instances and their rows.<br>In essence, a Tag is a vital part of Slick&#39;s internal mechanism for managing database tables and their rows, ensuring that each table instance is uniquely identifiable.<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>============================================row , tuple----------------------------------------------------------------------------------------<br><br>==========================================================pick later==========================================================<br>        this is the main method<br>          def getProductCatalogToPropertyDataFrame(parsedDF: DataFrame, bucketName: String)(implicit sparkSession: SparkSession, dbDetails: DbDetails, dbMap: Map[String, String], schemaName: String) = {<br><br>    log.info(&quot;Started Transformations and Processing of the Data | getProductCatalogToPropertyDataFrame()&quot;, customMeta)<br><br>    try {<br><br>      val activePostGresRecords = getPreviousDaySnapshot(dbDetails, dbMap, schemaName)<br><br>      // Drop the columns which are not required.<br>      val postGresDataDF = activePostGresRecords<br>        .drop(PcptSk,<br>          PcptEffEndDate,<br>          PcptActiveInd,<br>          PcptStatusDate,<br>          PcptInsertedBy,<br>          PcptInsertedOn)<br><br>      import sparkSession.implicits._<br>      val parsedProductCatalogDataDF = parsedDF<br>        .drop(DeflaggedDate,<br>          PeopleSftId,<br>          Province,<br>          Market,<br>          MrwdsParticipate,<br>          NonbrandedFlag,<br>          OpsRegion,<br>          GlbDivision,<br>          GlbRegion,<br>          AvpCode,<br>          AvpDescription,<br>          AkCategory,<br>          PrestigeTierCat,<br>          SeasStartDt1,<br>          SeasEndDt1,<br>          SeasDiscSch1,<br>          SeasDiscPert1,<br>          SeasRecId1,<br>          SeasSortId1,<br>          SeasStartDt2,<br>          SeasEndDt2,<br>          SeasDiscSch2,<br>          SeasDiscPert2,<br>          SeasRecId2,<br>          SeasSortId2,<br>          SeasStartDt3,<br>          SeasEndDt3,<br>          SeasDiscSch3,<br>          SeasDiscPert3,<br>          SeasRecId3,<br>          SeasSortId3,<br>          SeasStartDt4,<br>          SeasEndDt4,<br>          SeasDiscSch4,<br>          SeasDiscPert4,<br>          SeasRecId4,<br>          SeasSortId4,<br>          SeasStartDt5,<br>          SeasEndDt5,<br>          SeasDiscSch5,<br>          SeasDiscPert5,<br>          SeasRecId5,<br>          SeasSortId5,<br>          CityTaxPert,<br>          CityTaxType,<br>          VatTaxPert,<br>          VatTaxType,<br>          Filler)<br>        .map(row =&gt; applyschema(row))<br><br>      //  Changing the precision of the decimal columns<br>      val decimalType = DataTypes.createDecimalType(7, 2)<br><br>      val productCatalogDF = parsedProductCatalogDataDF<br>        .withColumn(LatSeconds + &quot;_TMP&quot;, col(LatSeconds).cast(decimalType))<br>        .withColumn(LonSeconds + &quot;_TMP&quot;, col(LonSeconds).cast(decimalType))<br>        .drop(LatSeconds, LonSeconds)<br>        .withColumnRenamed(LatSeconds + &quot;_TMP&quot;, LatSeconds)<br>        .withColumnRenamed(LonSeconds + &quot;_TMP&quot;, LonSeconds)<br>        .withColumn(OWNER, substring(col(OWNER), 1, 40))<br>        .withColumn(OWNER, regexp_replace(col(OWNER), &quot;,&quot;, &quot; &quot;))<br><br>      // Filtering out unwanted product codes.<br>      val requiredPropertiesDF = productCatalogDF.filter(!(col(ProductCode) isin(&quot;ST&quot;, &quot;GX&quot;)))<br><br>      //    Joining(Left Outer) current data with the postgreSQL db data on property number<br>      //    and dropping columns which are newly created in the current process<br>      val leftJoinedDFWithActiveRecDF = requiredPropertiesDF.join(postGresDataDF,<br>        requiredPropertiesDF(PropertyNum) === postGresDataDF(PcptPropertyNum) and<br>          requiredPropertiesDF(PropertyCode) === postGresDataDF(PcptPropertyCode), &quot;left_outer&quot;)<br><br>      // Getting the records which have undergone changes with respect to the database records.<br>      val updateDF = getUpdateRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Getting the records which does not match with database records that is which are new records.<br>      val insertDF = getInsertRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Getting the delete records.<br>      val deleteDF = getDeteteRecords(leftJoinedDFWithActiveRecDF)<br><br>      // Appending all the update,insert and delete records and adding few generic columns.<br>      val actionableDF = getUnionUpdateInsertDeleteDFs(updateDF, insertDF, deleteDF)<br><br>      // Dropping the database columns from the dataframe.<br>      val requiredColumnsDF = getRequiredColumnsDF(actionableDF)<br><br>      //    Renaming the input file columns to the database columns and creating the Json record<br>      //    which will be pushed to kafka required for ProductCatalogPropertyUpdateTargetJob*/<br>      val finalDataDF = getFinalDataFrame(requiredColumnsDF)<br><br>      //    Adding IsRecoverable column to differentiate recoverable, irrecoverable and valid data in the end to push to respective topics<br>      //    Making Recoverable as default, so that any exception occurs while dbconnection all the records will be pushed to error topic<br>      val persistDF = finalDataDF.withColumn(IsRecoverable, lit(Recoverable))<br><br>      //    Persisting the dataframe to database and respective topics<br>      persistDataFrame(persistDF, checkPointDirectory, bucketName, jobName)<br>    }<br>    catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;getProductCatalogToPropertyDataFrame() :: Exception while interacting with DB :: Exception: &quot; + ex.printStackTrace(), customMeta)<br>        val recoverableDF = parsedDF.coalesce(5)<br>          .select(from_json(col(SourceString), Encoders.product[EEHData].schema).as(Value))<br>        produceRecoverableMessage(recoverableDF)<br>    }<br><br>  }<br><br>   THIS IS THE MODIFIED TEST CASE WITH UPDATED SCHEMA FOR THIS<br><br>    test(&quot;getFinalDataFrame should return DataFrame with prefixed columns and JSON data column&quot;) {<br>    // Mock data for testing<br>    val spark: SparkSession = SparkSession.builder()<br>      .appName(&quot;Unit Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br>    val data = Seq(<br>     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;name1&quot;, &quot;status1&quot;, &quot;timeDiff1&quot;, &quot;active1&quot;, &quot;2021-01-01&quot;, &quot;2021-12-31&quot;, &quot;insertedBy1&quot;, &quot;2021-01-01&quot;, &quot;action1&quot;, &quot;owner1&quot;)<br>    )<br><br>    val schema = StructType(List(<br>     StructField(&quot;sourceString&quot;, StringType, false),<br>     StructField(&quot;PROP_NUM&quot;, IntegerType, false),<br>     StructField(&quot;PROP_CODE&quot;, StringType, false),<br>     StructField(&quot;UNIT&quot;, StringType, false),<br>     StructField(&quot;HGA_UNIT&quot;, StringType, false),<br>     StructField(&quot;PROP_NAME&quot;, StringType, false),<br>     StructField(&quot;PHYS_CAP&quot;, StringType, false),<br>     StructField(&quot;PRODUCT_INDEX&quot;, StringType, false),<br>     StructField(&quot;PRODUCT_CODE&quot;, StringType, false),<br>     StructField(&quot;REGION&quot;, StringType, false),<br>     StructField(&quot;GEO_REGION&quot;, StringType, false),<br>     StructField(&quot;HOTEL_TYPE&quot;, StringType, false),<br>     StructField(&quot;BILLING_TYPE&quot;, StringType, false),<br>     StructField(&quot;MGT_TYPE&quot;, StringType, false),<br>     StructField(&quot;NETWORK&quot;, StringType, false),<br>     StructField(&quot;PMS_TYPE&quot;, StringType, false),<br>     StructField(&quot;INTRFACE&quot;, StringType, false),<br>     StructField(&quot;ADDR1&quot;, StringType, false),<br>     StructField(&quot;ADDR2&quot;, StringType, false),<br>     StructField(&quot;CITY&quot;, StringType, false),<br>     StructField(&quot;STATE&quot;, StringType, false),<br>     StructField(&quot;ZIP_CODE&quot;, StringType, false),<br>     StructField(&quot;COUNTRY_CODE&quot;, StringType, false),<br>     StructField(&quot;CURRENCY_CODE&quot;, StringType, false),<br>     StructField(&quot;OPEN_DATE&quot;, StringType, false),<br>     StructField(&quot;COMP_FLAG&quot;, StringType, false),<br>     StructField(&quot;COMP_DT&quot;, StringType, false),<br>     StructField(&quot;ADD_DT&quot;, StringType, false),<br>     StructField(&quot;TIME_DIFF&quot;, StringType, false),<br>     StructField(&quot;DOLLARS&quot;, StringType, false),<br>     StructField(&quot;YMS_USER&quot;, StringType, false),<br>     StructField(&quot;ALT_ROUTE&quot;, StringType, false),<br>     StructField(&quot;ALT_ROUTE_FLAG&quot;, StringType, false),<br>     StructField(&quot;DBDB&quot;, StringType, false),<br>     StructField(&quot;KING&quot;, StringType, false),<br>     StructField(&quot;DOM_FRGN_PROP&quot;, StringType, false),<br>     StructField(&quot;INV_TYPE_YYPP&quot;, StringType, false),<br>     StructField(&quot;INV_TYPE&quot;, StringType, false),<br>     StructField(&quot;DIV&quot;, StringType, false),<br>     StructField(&quot;SHOPPE&quot;, StringType, false),<br>     StructField(&quot;ACCT_NUM&quot;, StringType, false),<br>     StructField(&quot;GROUP_ACCT&quot;, StringType, false),<br>     StructField(&quot;AMB_CODE&quot;, StringType, false),<br>     StructField(&quot;CORP_NUM&quot;, StringType, false),<br>     StructField(&quot;PHONE&quot;, StringType, false),<br>     StructField(&quot;FAX&quot;, StringType, false),<br>     StructField(&quot;AVAIL_CAP&quot;, StringType, false),<br>     StructField(&quot;SMSA&quot;, StringType, false),<br>     StructField(&quot;TRADING_AREA&quot;, StringType, false),<br>     StructField(&quot;LAT_DEGREES&quot;, StringType, false),<br>     StructField(&quot;LAT_MINUTES&quot;, StringType, false),<br>     StructField(&quot;LAT_SECONDS&quot;, StringType, false),<br>     StructField(&quot;LAT_N_OR_S&quot;, StringType, false),<br>     StructField(&quot;LON_DEGREES&quot;, StringType, false),<br>     StructField(&quot;LON_MINUTES&quot;, StringType, false),<br>     StructField(&quot;LON_SECONDS&quot;, StringType, false),<br>     StructField(&quot;LON_E_OR_W&quot;, StringType, false),<br>     StructField(&quot;X_PL_DIST_CD&quot;, StringType, false),<br>     StructField(&quot;REWARDS_CATEGORY&quot;, StringType, false),<br>     StructField(&quot;PROVINCE_CD&quot;, StringType, false),<br>     StructField(&quot;STATUS&quot;, StringType, false),<br>     StructField(&quot;STATUS_DT&quot;, StringType, false),<br>     StructField(&quot;TIME_OF_RECEIPT&quot;, StringType, false),<br>     StructField(&quot;ACTIVE_IND&quot;, StringType, false),<br>     StructField(&quot;EFF_START_DT&quot;, StringType, false),<br>     StructField(&quot;EFF_END_DT&quot;, StringType, false),<br>     StructField(&quot;LAST_UPD_BY&quot;, StringType, false),<br>     StructField(&quot;LAST_UPD_TS&quot;, StringType, false),<br>     StructField(&quot;ACTION_IND&quot;, StringType, false),<br>     StructField(&quot;OWNER&quot;, StringType, false)<br>    ))<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br>    )<br><br>    val resultDF = getFinalDataFrame(inputDF)<br><br>    // Basic assertions<br>    assert(resultDF != null, &quot;Resulting DataFrame should not be null&quot;)<br>//    assert(resultDF.columns.contains(ColumnPrefix + &quot;PCPT_SOURCE_STRING&quot;), s&quot;Resulting DataFrame should contain the column ${ColumnPrefix}PCPT_SOURCE_STRING&quot;)<br>//    assert(resultDF.columns.contains(ColumnPrefix + &quot;PCPT_PROPERTY_NUM&quot;), s&quot;Resulting DataFrame should contain the column ${ColumnPrefix}PCPT_PROPERTY_NUM&quot;)<br>//    assert(resultDF.columns.contains(JsonData), s&quot;Resulting DataFrame should contain the column $JsonData&quot;)<br>   }<br><br>ARRANGE THE DATA ACCORDINGLY TO THE SCHEMA AND IF YOU SEE ANY ISSUE IN SCHEMA LET ME KNOW<br><br><br><br><br><br><br><br>getting this error<br>GETTING THIS ISSUE C:\Users\ajune750\product-to-catalog\ProductCatalog-to-PropertyLookupETL\ProductCatalogToProperty\src\test\scala\com\marriott\mgpapp\spark\productcatalog\property\helper\ProductCatalogToPropertyJobHelperTest.scala:1013:6<br>too many elements for tuple: 70, allowed: 22<br>     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br><br>     solution:<br><br>     The error you&#39;re encountering is because Scala tuples have a maximum size limit of 22 elements. Since your data has 70 elements, you need to use a different approach to create the DataFrame.<br><br>You can use a Row object instead of a tuple to represent each row of data. Here&#39;s how you can adjust your test case:<br><br>Use Row objects to represent the data.<br>Create the DataFrame from the list of Row objects and the schema.<br>Here&#39;s the modified test case:<br><br>     //    val data = Seq(<br>//     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>//    )<br>val data = Seq(<br> Row(&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>)<br><br><br><br>C:\Users\ajune750\product-to-catalog\ProductCatalog-to-PropertyLookupETL\ProductCatalogToProperty\src\test\scala\com\marriott\mgpapp\spark\productcatalog\property\helper\ProductCatalogToPropertyJobHelperTest.scala:1092:50<br>type mismatch;<br> found   : Product =&gt; org.apache.spark.sql.Row<br> required: org.apache.spark.sql.Row =&gt; ?<br>     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br><br><br><br><br>The error you&#39;re encountering is because Scala tuples have a maximum size limit of 22 elements. Since your data has 70 elements, you need to use a different approach to create the DataFrame.<br><br>You can use a Row object instead of a tuple to represent each row of data. Here&#39;s how you can adjust your test case:<br><br>Use Row objects to represent the data.<br>Create the DataFrame from the list of Row objects and the schema.<br>Here&#39;s the modified test case:<br><br><br><br>=========================================================================================================================================================================<br><br><br><br><br><br><br><br><br><br><br>builder object ki zarrorrat kuon hain<br>hum kya mock kar skate hai<br>kya nahi <br><br>class ko nahi mock kar skate <br><br><br>individual ab connection ko mock kar sakte hai<br><br><br>function ke ander ek aur object bana ho connection ka usko mock nahi kar sakte <br><br><br>uske liye 2 solution hai<br><br><br>ya to builderutil me object bana do<br><br><br>ya jaise collation me kiya tha usko pass kara do parameter me <br><br><br><br><br>builder util ka example rateplan se<br><br>and <br><br><br><br>balki doo example mil jayenge<br><br>ek builder util wala bhi and paramter me pass karane wala bhi collation me<br><br><br>===================================================================================================================<br><br><br><br>foreachbatch not mockable<br><br>how to do it what is the method lalit used<br><br>singelton builder class me <br><br>kuonki<br><br>bar baar call karte hai<br><br>kuonki wo object hota hai<br>aur hum class ko nahi kar sakte call aur mock<br><br>singlteon ka ek hi instance hota hai<br><br>aur uska object crreste naho hota<br><br><br><br>=========================================================================================================<br>for converting a dataframe sparksession is must<br><br>   Create a SparkSession for testing<br>   val spark: SparkSession = SparkSession.builder()<br>     .appName(&quot;UnitTest&quot;)<br>     .master(&quot;local[*]&quot;)<br>     .getOrCreate()<br><br>===================================================================<br><br>         import spark.implicits._<br><br>    // Sample input data<br>    val inputData = Seq(<br>     (&quot;value1&quot;, &quot;payload1&quot;)<br>   , (&quot;value2&quot;, &quot;payload2&quot;)<br>    ).<br>     // toDF(&quot;someColumn&quot;)<br>      toDF(&quot;someColumn&quot;, &quot;payLoad&quot;)<br><br><br>      ========================================================================<br><br><br>how to make  dataframe <br>and what is tuple exceeds 22 then we nee to covert a row <br><br>parallelize also changes<br><br>       val spark: SparkSession = SparkSession.builder()<br>      .appName(&quot;Unit Test&quot;)<br>      .master(&quot;local[*]&quot;)<br>      .getOrCreate()<br><br>    val data = Seq(<br>     Row(&quot;event1&quot;),<br>     Row(&quot;ea oent2&quot;)<br>    )<br><br>    val schema = StructType(List(<br>     StructField(Value, StringType, false)<br>    ))<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data),<br>     schema<br>    )<br><br>    val resultDF = transformNonConvertedEntitiesToIrrecoverableDLTMessage(inputDF)<br>    assert(resultDF != null, &quot;Resulting DataFrame should not be null&quot;)<br>   }<br><br>   differentiate<br><br>   //    val data = Seq(<br>//     (&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>//    )<br>val data = Seq(<br> Row(&quot;source1&quot;, 1, &quot;code1&quot;, &quot;unit1&quot;, &quot;hgaUnit1&quot;, &quot;name1&quot;, &quot;physCap1&quot;, &quot;productIndex1&quot;, &quot;productCode1&quot;, &quot;region1&quot;, &quot;geoRegion1&quot;, &quot;hotelType1&quot;,&quot;mktg_HOTEL_TYP1&quot;, &quot;billingType1&quot;, &quot;mgtType1&quot;, &quot;network1&quot;, &quot;pmsType1&quot;, &quot;interface1&quot;, &quot;addr1&quot;, &quot;addr2&quot;, &quot;city1&quot;, &quot;state1&quot;, &quot;zip1&quot;, &quot;countryCode1&quot;, &quot;currencyCode1&quot;, &quot;2021-01-01&quot;, &quot;compFlag1&quot;, &quot;2021-01-01&quot;, &quot;2021-01-01&quot;, &quot;timeDiff1&quot;, &quot;dollars1&quot;, &quot;ymsUser1&quot;, &quot;altRoute1&quot;, &quot;altRouteFlag1&quot;, &quot;dbdb1&quot;, &quot;king1&quot;, &quot;domFrgnProp1&quot;, &quot;invTypeYypp1&quot;, &quot;invType1&quot;, &quot;div1&quot;, &quot;shoppe1&quot;, &quot;acctNum1&quot;, &quot;groupAcct1&quot;, &quot;ambCode1&quot;, &quot;corpNum1&quot;, &quot;phone1&quot;, &quot;fax1&quot;, &quot;availCap1&quot;, &quot;smsa1&quot;, &quot;tradingArea1&quot;, &quot;latDegrees1&quot;, &quot;latMinutes1&quot;, &quot;latSeconds1&quot;, &quot;latNOrS1&quot;, &quot;lonDegrees1&quot;, &quot;lonMinutes1&quot;, &quot;lonSeconds1&quot;, &quot;lonEOrW1&quot;, &quot;xPlDistCd1&quot;, &quot;rewardsCategory1&quot;, &quot;provinceCd1&quot;, &quot;status1&quot;, &quot;statusDt1&quot;, &quot;timeOfReceipt1&quot;, &quot;activeInd1&quot;, &quot;effStartDt1&quot;, &quot;effEndDt1&quot;, &quot;lastUpdBy1&quot;, &quot;lastUpdTs1&quot;, &quot;actionInd1&quot;, &quot;owner1&quot;)<br>)<br><br>then this as well<br><br>//    val inputDF = spark.createDataFrame(<br>//     spark.sparkContext.parallelize(data.map(Row.fromTuple)), schema<br>//    )<br><br>    val inputDF = spark.createDataFrame(<br>     spark.sparkContext.parallelize(data), schema<br>    )<br><br><br><br><br><br><br><br><br><br>    ===========================================================================================================<br><br><br>    sqlcontext.reader for query to read<br><br>    also map(ke ander jo query hoti hai wo daalni hoti hai agar error aye db map me to) key is missing<br><br>     test(&quot;getPreviousDaySnapshot should fetch snapshot data from PostgreSQL DB&quot;) {<br>    implicit val sparkSession: SparkSession = mock[SparkSession]<br>    val sqlContext = mock[org.apache.spark.sql.SQLContext]<br>    //newly added  reader for null pointer exception<br>    val reader = mock[org.apache.spark.sql.DataFrameReader]<br>    val dbDetails = DbDetails(<br>     jdbcurl = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivername = &quot;org.postgresql.Driver&quot;,<br>     username = &quot;testuser&quot;,<br>     password = &quot;testpass&quot;,<br>     poolsize = &quot;5&quot;,<br>     idletimeout = 1000L,<br>     querytimeout = scala.concurrent.duration.Duration.Inf,<br>     jdbcurlro = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivernamero = &quot;org.postgresql.Driver&quot;,<br>     usernamero = &quot;testuserro&quot;,<br>     passwordro = &quot;testpassro&quot;,<br>     poolsizero = &quot;5&quot;,<br>     idletimeoutro = 1000L,<br>     querytimeoutro = scala.concurrent.duration.Duration.Inf,<br>     jdbcurlwo = &quot;jdbc:postgresql://localhost:5432/testdb&quot;,<br>     drivernamewo = &quot;org.postgresql.Driver&quot;,<br>     usernamewo = &quot;testuserwo&quot;,<br>     passwordwo = &quot;testpasswo&quot;,<br>     poolsizewo = &quot;5&quot;,<br>     idletimeoutwo = 1000L,<br>     querytimeoutwo = scala.concurrent.duration.Duration.Inf<br>    )<br>    val dbMap = Map(ProductCatalogProperty -&gt; &quot;product_catalog&quot;)<br>    val schemaName = &quot;public&quot;<br><br><br>    // Mocking SparkSession behavior<br>    when(sparkSession.sqlContext).thenReturn(sqlContext)<br>    when(sqlContext.read).thenReturn(reader)<br><br>    // Mocking DataFrame<br>    val postGresDataDB: DataFrame = mock[DataFrame]<br>    val postGresDataDF: DataFrame = mock[DataFrame]<br>    val activePostGresRecords: DataFrame = mock[DataFrame]<br><br>//    when(sqlContext.read.jdbc(anyString(), anyString(), any[java.util.Properties])).thenReturn(postGresDataDB)<br>    when(reader.jdbc(anyString(), anyString(), any[java.util.Properties])).thenReturn(postGresDataDB)<br>//    java.lang.NullPointerException was thrown.<br>    when(postGresDataDB.columns).thenReturn(Array(&quot;pcpt_active_ind&quot;, &quot;other_column&quot;))<br>   // when(postGresDataDB.toDF(anyVararg[String])).thenReturn(postGresDataDF)<br>   // not found: value anyVararg<br>   // when(postGresDataDF.filter(any())).thenReturn(activePostGresRecords)<br>//    getting this ambiguous reference to overloaded definition,<br>//    both method filter in class Dataset of type (func: org.apache.spark.sql.Row =&gt; Boolean)org.apache.spark.sql.Dataset[org.apache.spark.sql.Row]<br>    when(postGresDataDB.toDF(any[Array[String]]: _*)).thenReturn(postGresDataDF)<br>    when(postGresDataDF.filter(any[org.apache.spark.sql.Column])).thenReturn(activePostGresRecords)<br>    val result = getPreviousDaySnapshot(dbDetails, dbMap, schemaName)<br>    assert(result != activePostGresRecords)<br>  }<br><br><br><br><br><br><br><br><br><br><br><br><br><br>=====================================================================================================================================================<br><br><br><br>sneha ne jo bataya tha kisi streaming job ko batch me convert karne ka tarika copy me note hai<br><br>notepad pe bhi milega<br><br>kanta ci_sgds folder me psinventory ke liye bhi hai<br><br><br><br>===================================================================================================================<br><br><br>notebbok me bahut kuch likha hai<br><br>and oozie scripts<br><br><br>====================================================================================================================<br><br>emr, spark submit, core nodes, cleaning the emr<br><br>=========================================================================================================================================================<br><br>100000 learnings of emr<br>===============================================================================================<br><br>all the tickets<br><br><br>==============================================================================<br><br>mapgroup<br><br>================================================<br><br>rate plan how to take the last variable in try and catch<br><br> try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while interacting with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          singleToRecoverable(recoverableDf)<br>          //sparkSession.emptyDataFrame<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>    lastDate<br><br><br>    ====================================================================================<br><br><br>    import from classs as well as object<br><br>    package object hai directly import ho jayega<br><br><br>    class hai to pehle uska object banao tabhi import kar sakte ho isliye testconstant ki class nahi import ho rhi thi<br>     val dbdetail2 = new TestConstants<br>    val result = getPreviousDaySnapshot(dbdetail2, dbMap, schemaName)<br><br>    assert(result == emptyDF)<br>   }<br><br><br><br><br>   =======================================================================================<br><br><br>   fetch fetch:  sari branches sync me ho jayegi<br><br>git checkout -b feture-mgp/47628<br><br>local and remote<br><br>teams chat also<br><br><br>git clone  https://git.marriott.com/RMS/ReservationETL.git development-mgp<br><br><br>====================================================<br><br>INTELLIJ MEMORY <br><br>2GB SE 4GB KARI HELP ME JAAKE<br> SO THAT JOB TAKES LESS TIME<br><br> ================================================================<br><br> test case outstanding work <br><br> utkarsaha ne isko upar method me hi daal diya hai taaki neeche wo mock karnw hi na pade properties ko jisse eerror aa raha hai<br><br><br> same tet case me bhi value pass kar di hai<br><br><br><br>  def produceOutputMessage(dataFrame: DataFrame, destTopic: String = KafkaProperties.destinationTopic): Unit = {<br>    try {<br>      if (dataFrame.isEmpty) {<br>        log.info(&quot;Received Empty DataFrame, not inserting any records to Destination Topic | produceOutputMessage()&quot;, customMeta)<br>      } else {<br>        prepareBatchProducer(dataFrame, destTopic, Some(classOf[EEHData]), Some(&quot;source_schema.avsc&quot;))<br>        log.info(&quot;Successfully inserted records to Destination Topic | produceOutputMessage()&quot;, customMeta)<br>      }<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;produceOutputMessage() :: Exception while producing records to Kafka Topic :: Exception:&quot; + exception.printStackTrace(), customMeta)<br>        new KafkaException(exception, errorMeta).handle()<br>    }<br>  }<br><br><br><br>  test case :<br><br>  produceOutputMessage(dataFrame, &quot;destination-test-topic&quot;)<br><br><br>   Caused by: java.io.FileNotFoundException: Properties file cannot be loaded<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelperImpl.init(PropertiesHelper.scala:32)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.init(PropertiesHelper.scala:85)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.&lt;init&gt;(PropertiesHelper.scala:84)<br>19:17:20  [error]  at com.marriott.rm.platform.util.PropertiesHelper$.&lt;clinit&gt;(PropertiesHelper.scala)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties.initKafkaConfig$default$1(KafkaProperties.scala:170)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties$.&lt;init&gt;(KafkaProperties.scala:288)<br>19:17:20  [error]  at com.marriott.rm.platform.constants.KafkaProperties$.&lt;clinit&gt;(KafkaProperties.scala)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.getStreamInstance(KafkaCircuitBreaker.scala:28)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.&lt;init&gt;(KafkaCircuitBreaker.scala:12)<br>19:17:20  [error]  at com.marriott.rm.platform.util.KafkaCircuitBreaker$.<br><br><br>----------------------------------------------------------------------------------<br><br>preparebatchproducer<br><br><br>/    val mockProducerHelper = mock[ProducerHelper[EEHData]]<br>//    when(mockProducerHelper.produce()).thenReturn(None)<br>//    when(<br>//     prepareBatchProducer(<br>//      ArgumentMatchers.any(classOf[DataFrame]),<br>//      ArgumentMatchers.any(classOf[String]),<br>//      ArgumentMatchers.eq(Option(classOf[EEHData])),<br>//      ArgumentMatchers.any()<br>//<br>//     )(ArgumentMatchers.any())<br>//    ).<br>////      thenReturn(mockProducerHelper)<br>//      thenReturn(None)<br><br>---------------------------------------------------------------------------------------------------------------------------<br><br><br>THIS IS RecoverableDf:Dataset[MergedRatePlan],<br><br>  case class MergedRatePlan(<br>                           ratePlan: RatePlan,<br>                           sourceString: String<br>                         ) HOW TO PRINT THE sourceString: Strin IN SCALA USING RecoverableDf <br>                         TELL ME ALL THE APPROACHES<br><br>                         ANS<br><br><br><br><br>To print the sourceString field from the MergedRatePlan case class in Scala using a DataFrame (RecoverableDf), you can use several approaches depending on your specific needs. Here are some of the common methods:<br><br>1. Using the show Method<br>If you want to display the DataFrame and include the sourceString column, you can use the show method:<br><br>RecoverableDf.show()<br>This will print the entire DataFrame to the console, including the sourceString column.<br><br>2. Selecting and Showing the sourceString Column<br>If you only want to print the sourceString column:<br><br>RecoverableDf.select(&quot;sourceString&quot;).show()<br>3. Collecting the Data and Iterating<br>You can collect the data to the driver and then iterate over the rows to print each sourceString:<br><br>RecoverableDf.select(&quot;sourceString&quot;).collect().foreach(row =&gt; println(row.getString(0)))<br>4. Using Dataset API<br>If RecoverableDf is already a Dataset of MergedRatePlan, you can use the Dataset API to extract and print the sourceString:<br><br>RecoverableDf.map(_.sourceString).foreach(println)<br>5. Using the rdd Method<br>You can convert the DataFrame to an RDD and then use RDD operations to print the sourceString:<br><br>RecoverableDf.select(&quot;sourceString&quot;).rdd.map(row =&gt; row.getString(0)).foreach(println)<br>6. Using SQL Queries<br>If you want to use SQL queries, you can register the DataFrame as a temporary view and run a SQL query:<br><br>RecoverableDf.createOrReplaceTempView(&quot;merged_rate_plan_view&quot;)<br>val sqlDF = spark.sql(&quot;SELECT sourceString FROM merged_rate_plan_view&quot;)<br>sqlDF.show()<br>7. Using withColumn Method<br>If you want to create a new DataFrame with only the sourceString column and then print it:<br><br>val sourceStringDf = RecoverableDf.withColumn(&quot;sourceString&quot;, col(&quot;sourceString&quot;))<br>sourceStringDf.show()<br>8. Using filter and select<br>If you have specific filtering criteria and then want to print the sourceString:<br><br>RecoverableDf.filter($&quot;sourceString&quot;.isNotNull).select(&quot;sourceString&quot;).show()<br>Remember to import the necessary Spark SQL and DataFrame functions:<br><br>import org.apache.spark.sql.SparkSession<br>import org.apache.spark.sql.functions._<br>Example Setup<br>Here is a small example setup to illustrate the usage:<br><br>import org.apache.spark.sql.{SparkSession, Dataset}<br><br>case class RatePlan(planName: String)<br>case class MergedRatePlan(ratePlan: RatePlan, sourceString: String)<br><br>val spark = SparkSession.builder.appName(&quot;Example&quot;).getOrCreate()<br>import spark.implicits._<br><br>// Sample data<br>val data = Seq(<br>  MergedRatePlan(RatePlan(&quot;PlanA&quot;), &quot;SourceA&quot;),<br>  MergedRatePlan(RatePlan(&quot;PlanB&quot;), &quot;SourceB&quot;)<br>)<br><br>// Creating DataFrame<br>val RecoverableDf: Dataset[MergedRatePlan] = data.toDS()<br><br>// Printing sourceString using different approaches<br>RecoverableDf.show()<br>RecoverableDf.select(&quot;sourceString&quot;).show()<br>RecoverableDf.select(&quot;sourceString&quot;).collect().foreach(row =&gt; println(row.getString(0)))<br>RecoverableDf.map(_.sourceString).foreach(println)<br>RecoverableDf.select(&quot;sourceString&quot;).rdd.map(row =&gt; row.getString(0)).foreach(println)<br>RecoverableDf.createOrReplaceTempView(&quot;merged_rate_plan_view&quot;)<br>val sqlDF = spark.sql(&quot;SELECT sourceString FROM merged_rate_plan_view&quot;)<br>sqlDF.show()<br>val sourceStringDf = RecoverableDf.withColumn(&quot;sourceString&quot;, col(&quot;sourceString&quot;))<br>sourceStringDf.show()<br>RecoverableDf.filter($&quot;sourceString&quot;.isNotNull).select(&quot;sourceString&quot;).show()<br>Choose the method that best fits your use case.<br><br><br>==============================================================================================================================<br><br><br><br><br><br><br><br><br><br><br><br><br><br>full article prepared**************************************************************************************************************************************<br><br>  this is the method,<br><br>                         def singleToRecoverable(inputDS: Dataset[MergedRatePlan])(implicit sparkSession: SparkSession): Unit = {<br>    log.info(&quot;entered into recoverable topic&quot;, customMeta)<br>    try {<br>      val finalDS = inputDS<br>        .select(col(&quot;sourceString&quot;))<br>        //.coalesce(5)<br>        .select(from_json(col(&quot;sourceString&quot;)<br>          , Encoders.product[com.marriott.mmf.RatePlan].schema).as(&quot;value&quot;))<br><br><br>this is line 120 (RatePlanIngestionHelper.scala:120) where its showing the below error <br><br>                               class:c.m.m.s.r.i.j.RatePlanIngestionJob - Exception while having a connect with DB :: Exception: () - <br>2024-09-16 15:54:42.083 [Executor task launch worker for task 163.0 in stage 15.0 (TID 1011)] INFO <br>                                class:c.m.m.s.r.i.j.RatePlanIngestionJob - entered into recoverable topic - org.apache.spark.SparkException:  Dataset transformations and actions can only be invoked by the driver, not inside of other Dataset transformations; for example, dataset1.map(x =&gt; dataset2.values.count() * x) is invalid because the values transformation and count action cannot be performed inside of the dataset1.map transformation. For more information, see SPARK-28702.       <br>  at org.apache.spark.sql.errors.QueryExecutionErrors$.transformationsAndActionsNotInvokedByDriverError(QueryExecutionErrors.scala:1967)<br>  at org.apache.spark.sql.Dataset.sparkSession$lzycompute(Dataset.scala:198)<br>  at org.apache.spark.sql.Dataset.sparkSession(Dataset.scala:196)<br>  at org.apache.spark.sql.Dataset.withPlan(Dataset.scala:3887)<br>  at org.apache.spark.sql.Dataset.select(Dataset.scala:1519)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.singleToRecoverable(RatePlanIngestionHelper.scala:120)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.singleToRecoverable$(RatePlanIngestionHelper.scala:116)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.jobs.RatePlanIngestionJob.singleToRecoverable(RatePlanIngestionJob.scala:20)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.$anonfun$persistDB$1(RatePlanIngestionHelper.scala:508)<br>  at com.marriott.mgpapp.spark.rateplan.ingestion.helper.RatePlanIngestionHelper.$anonfun$persistDB$1$adapted(RatePlanIngestionHelper.scala:499)<br>  at scala.collection.Iterator.foreach(Iterator.scala:943)<br><br><br><br>  this is the method from where singleToRecoverable  is invoked<br>   def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf:Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit  sparkSession: SparkSession,jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]) = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;,<br>      customMeta)<br>    ratePlanBookingLimitDS.foreach { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br><br>        dbSinkObject.process(row)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          singleToRecoverable(recoverableDf)<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>    lastDateDF.foreach{dates=&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>        //delete stay_date &gt; last_date<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      }<br>      finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>  }<br>}<br><br><br><br> def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    val failedRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(row) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>    if (failedRows.nonEmpty) {<br>      singleToRecoverable(recoverableDf)<br>    }<br><br>    lastDateDF.foreach { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }<br>  }<br>}<br><br><br>we need to pass the same singleToRecoverable(recoverableDf) in the catch block if we find the exception for this  lastDateDF.foreach { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>      } finally {<br>        dbSinkObject.close(null)<br>      }, now give me the modified persistDB function according to that. <br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>      the final solution:<br><br><br><br><br><br><br><br>      ===========approcah 1=============================================================================================================<br><br><br><br>     To resolve this issue, you need to ensure that singleToRecoverable is invoked from the driver rather than within the foreach action. One approach is to collect the data to the driver and then process it, or handle the exceptions differently.<br><br>Approach 1: Collect to Driver and Process<br>This approach involves collecting the Dataset to the driver and then processing each element. This ensures that singleToRecoverable is invoked on the driver side.<br><br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  val ratePlanBookingLimitArray = ratePlanBookingLimitDS.collect() // Collect to driver<br>  ratePlanBookingLimitArray.foreach { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        singleToRecoverable(recoverableDf) // This is now safe<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  val lastDateArray = lastDateDF.collect() // Collect to driver<br>  lastDateArray.foreach { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br>}<br><br><br><br><br><br><br><br><br><br><br>==========================approach 2=======================================================================<br><br>Approach 2: Handle Exceptions Differently<br>Another approach is to handle the exceptions such that singleToRecoverable is not invoked directly within the foreach action. Instead, you can collect the failed rows and process them separately on the driver.<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>====================question----------------------------------------------------------------------<br><br>in the above case why we have chaned foreach to flatmap they both are transformations and most probably same error will occur in bot the cases<br><br>Copy<br><br><br>answer--------------------------------------------------------------------------------------------------------------<br><br>You are correct that both foreach and flatMap are transformations; however, the crucial difference lies in how they are used and where the actions are triggered. The objective is to ensure that Spark operations (transformations and actions) are not nested within actions. <br><br>In the previous solution, flatMap was used to accumulate failed rows, and the collection of these rows happens as a separate action (collect), which is performed at the driver level. This ensures that singleToRecoverable is invoked on the driver.<br><br>However, if flatMap doesn&#39;t seem to resolve the issue and the error persists, we need to ensure that no Spark transformations or actions are nested within actions. <br><br>Refactoring the Code:<br>Let&#39;s ensure that all Spark operations are executed at the driver level by collecting the entire Dataset to the driver before processing. Here&#39;s a revised approach:<br><br>Collect the ratePlanBookingLimitDS and lastDateDF to the driver.<br>Process each row on the driver.<br>Handle exceptions and invoke singleToRecoverable appropriately.<br><br><br><br>======================================================================<br><br><br>final<br><br><br>question==============================================================================<br><br>just confirming this wont impact the succesfull scenario where it does not catch  the error  as we have changed from foreach to flatmap<br><br>Copy================answer========================================================<br><br>Switching from foreach to flatMap in the context provided earlier should not impact the successful scenario where no errors are encountered. However, there is a subtle difference in how these two operations work:<br><br>foreach: Executes a function for each element in the Dataset. This is an action and triggers the computation.<br>flatMap: Applies a function to each element in the Dataset and flattens the results. This is a transformation and is lazy until an action is called on the resulting Dataset.<br><br><br>Ensuring No Impact on Successful Scenario:<br>The main goal is to ensure that we handle the data processing correctly and only invoke singleToRecoverable when an error occurs. The refactored approach should work seamlessly for both successful and error scenarios because:<br><br>Data Processing: Data is processed using foreach after collecting it to the driver, ensuring that each row is handled individually.<br>Error Handling: Errors are captured and used to set a flag, which controls whether singleToRecoverable is invoked.<br>Control Flow: The control flow remains the same for successful scenarios—each row is processed, and singleToRecoverable is only called if an error occurs.<br><br><br>Final Code with Explanation:<br>Let&#39;s revalidate the final code to ensure it correctly handles both successful and error scenarios.<br><br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  // Collect the datasets to the driver<br>  val ratePlanBookingLimitArray = ratePlanBookingLimitDS.collect() <br>  val lastDateArray = lastDateDF.collect()<br><br>  var hasExceptionOccurred = false<br><br>  // Process ratePlanBookingLimitArray on the driver<br>  ratePlanBookingLimitArray.foreach { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  // Process lastDateArray on the driver<br>  lastDateArray.foreach { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }<br><br>  // If any exception occurred, invoke singleToRecoverable<br>  if (hasExceptionOccurred) {<br>    singleToRecoverable(recoverableDf)<br>  }<br>}<br>Explanation:<br><br>Collecting Data: <br>ratePlanBookingLimitDS.collect() and lastDateDF.collect() ensure all data is available on the driver, avoiding any distributed processing issues.<br><br><br>Processing Data: <br>ratePlanBookingLimitArray.foreach { row =&gt; ... } and lastDateArray.foreach { dates =&gt; ... } process each element individually. This is the same logic that was previously inside the foreach on the Dataset but now executes on the driver.<br><br><br>Error Handling: <br>Errors are captured, and a flag (hasExceptionOccurred) is set to true if any error occurs.<br><br><br>Control Flow: <br>If no errors occur, hasExceptionOccurred remains false, and singleToRecoverable is not invoked.<br>If any error occurs, hasExceptionOccurred is set to true, and singleToRecoverable(recoverableDf) is invoked after processing all rows.<br><br><br>Conclusion:<br>This approach ensures that the successful scenario where no errors are encountered will not be impacted. The data processing logic remains the same, and singleToRecoverable is only invoked when necessary, based on the error flag. This maintains the integrity of the original logic while adhering to Spark&#39;s execution model.<br><br><br>Copy<br><br><br><br><br>----------------------------------<br><br>this is the method:<br><br><br>  def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.process(row)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(row) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>    // Process lastDateDF and collect failed rows<br>    val failedDateRows = lastDateDF.flatMap { dates =&gt;<br>      val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>      try {<br>        dbSinkObject.open(1.toLong, 2)<br>        dbSinkObject.deleteRecords(dates)<br>        None<br>      } catch {<br>        case exception: Exception =&gt;<br>          log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>          Some(dates) // Collect the failed row<br>      } finally {<br>        dbSinkObject.close(null)<br>      }<br>    }(Encoders.product[RatePlanBookingLimitDeletion]).collect() // Collect the failed rows to the driver<br><br>    // Invoke singleToRecoverable if there are any failed rows<br>    if (failedRatePlanRows.nonEmpty || failedDateRows.nonEmpty) {<br>      singleToRecoverable(recoverableDf)<br>    }<br>  }<br><br><br><br><br>  ============================changed to the new one<br><br><br><br>  COLLECT KIYA ...DATA DRIVER PE AA GYA<br><br>  DRIVER RETURN KAR DIYA<br><br>  256MB PE FAIL KAR JAYEGI<br><br><br>  COLLECT KARTE HAI HUM ...<br><br><br><br>  REPARTITION KARTE 5 <br><br>  PER  EXECUTIONER PARTITIONING KE LIYE<br><br>  100 EVENT HAI EXECUTER , 5 SE DIVIDE KARKE 20 HOGAYA<br>  <br>  TO WO EK <br><br><br><br><br>*******************************************************************************************************************************************************************<br>def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>              lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>              recoverableDf: Dataset[MergedRatePlan],<br>              schemaName: String,<br>              tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>  log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>  // Process ratePlanBookingLimitDS and collect failed rows<br>  val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        Some(row) // Collect the failed row<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }(Encoders.product[RatePlanBookingLimit]).collect() // Collect the failed rows to the driver<br><br>  // Process lastDateDF and collect failed rows<br>  val failedDateRows = lastDateDF.flatMap { dates =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>    try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.deleteRecords(dates)<br>      None<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>        Some(dates) // Collect the failed row<br>    } finally {<br>      dbSinkObject.close(null)<br>    }<br>  }(Encoders.product[RatePlanBookingLimitDeletion]).collect() // Collect the failed rows to the driver<br><br>  // Invoke singleToRecoverable if there are any failed rows<br>  if (failedRatePlanRows.nonEmpty || failedDateRows.nonEmpty) {<br>    singleToRecoverable(recoverableDf)<br>  }<br>}<br><br>for the above solution you provided please put foreach and dont replace it with flatmap, also dont use collect as it in not optimised<br>you can do one thing you can start the try block  from the start of the method and hencforth adjust catch block as well<br><br><br>like for example<br><br>dont start from here  try {<br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    }  rather you can start from here<br><br><br> try {<br>    val failedRatePlanRows = ratePlanBookingLimitDS.flatMap { row =&gt;<br>    val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>   <br>      dbSinkObject.open(1.toLong, 2)<br>      dbSinkObject.process(row)<br>      None<br>    }<br><br><br><br><br>    =====================================final solution===============================================================================<br><br>      def persistDB(ratePlanBookingLimitDS: Dataset[RatePlanBookingLimit],<br>                lastDateDF: Dataset[RatePlanBookingLimitDeletion],<br>                recoverableDf: Dataset[MergedRatePlan],<br>                schemaName: String,<br>                tableName: String)(implicit sparkSession: SparkSession, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails, checkpointPath: Option[String]): Unit = {<br>    log.info(&quot;Persisting the Dataset - persistDS()&quot;, customMeta)<br><br>    var hasExceptionOccurred = false // Flag to track if any exception occurs<br><br>    try {<br>      // Process ratePlanBookingLimitDS using foreach<br>      ratePlanBookingLimitDS.foreach { row =&gt;<br>        val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>        try {<br>          dbSinkObject.open(1.toLong, 2)<br>          dbSinkObject.process(row)<br>        } catch {<br>          case exception: Exception =&gt;<br>            log.error(&quot;Exception while having a connect with DB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>            hasExceptionOccurred = true<br>        } finally {<br>          dbSinkObject.close(null)<br>        }<br>      }<br><br>      // Process lastDateDF using foreach<br>      lastDateDF.foreach { dates =&gt;<br>        val dbSinkObject = new RatePlanIngestionDbSink(schemaName, tableName)<br>        try {<br>          dbSinkObject.open(1.toLong, 2)<br>          dbSinkObject.deleteRecords(dates)<br>        } catch {<br>          case exception: Exception =&gt;<br>            log.error(&quot;Exception OCCURED WHILE CONNECTING TO dB :: Exception: &quot; + exception.printStackTrace(), customMeta)<br>            hasExceptionOccurred = true<br>        } finally {<br>          dbSinkObject.close(null)<br>        }<br>      }<br><br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception encountered during persistDB execution: &quot; + exception.printStackTrace(), customMeta)<br>        hasExceptionOccurred = true<br>    }<br><br>    // If any exception occurred, invoke singleToRecoverable<br>    if (hasExceptionOccurred) {<br>      singleToRecoverable(recoverableDf)<br>    }<br>  }<br><br>  <br><br><br>  ***********************************************************************************************************************************************************************************************************88reservation  ingestion how to give sparksession and df empty in return<br><br> catch {<br>      case exception: Exception =&gt;<br>        new ExceptionHandler(exception).handle(<br>          Map(<br>            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>            jobName -&gt; jobConfigDetails.jobname,<br>            customErrorMessage -&gt; {<br>              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records&quot;&quot;&quot;.stripMargin<br>            }<br>          )<br>        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br><br>    }<br><br><br><br>    --------------------==============================================================================================================================================<br><br><br>    merge conflicts<br><br><br>    pehle to uncommit kiya <br>    usse pehle stash kana zaruri hai<br><br>    uncommit ho jata hai<br>    sourcetree se=============================================<br><br><br>    ab merge conflict aaye<br><br><br>    pehle to apni branch me jao<br><br>    fir git pull dev-mgp<br><br><br>    ab issme intellij pe aa gaye merge conflict unko resolve karo<br><br><br>    ==============================================<br><br><br>    git pull me sari dusre folder bhi stage ho jate hai<br><br>    isliye git rebase karna chahiye wo directly merge nahi karta hua<br><br><br><br>    simple use sourcetree<br><br>    =================================================================<br><br>    <br><br><br><br><br><br><br>     import sparkSession.implicits._<br><br>     without this was getting the error <br><br>         val payloadEvent = filteredReservationDS.map(x =&gt; x.reservationPayload.payLoad).toDF<br><br>error : no implicit found for paramter  encoder<br><br><br>===================================================================================================================<br><br><br>two methods different catch exceptions<br><br><br>ist one:<br><br> def fetchPMSfromRateplan(reservationRecordsList : List[ReservationCollationEvent])<br>                          (implicit jobConfigurationDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails):List[ReservationCollationEvent]={<br>    try {<br><br>      //    var replacement = &quot;Corporate&quot;<br>      val updatedList = reservationRecordsList.map { row =&gt;<br>        val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br>          val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>            if(!(ratePlan.marketType.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.isDefined &amp;&amp; ratePlan.marketType.get.marketCode.get.nonEmpty)) { // --  if pms is empty then fetch it&#39;s value from MongoDB<br>              val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>              val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) //--  2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>              val mongodbClient = getMongoClient(conf, DocumentDBInput)<br>              val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>              val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>              val key = &quot;RATEDEFNDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + ratePlan.code<br>              val regexPattern = s&quot;&quot;&quot;^$key.*$$&quot;&quot;&quot;.r.pattern<br>              val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>              val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>              val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>              //            val filter1 = Document(&quot;_id&quot; -&gt; &quot;RATEDFENDOC_BOGAR_KAR001_2024-04-30 13:07:18_&quot;)<br>              val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>              rateplanupdated<br>            } else {<br>              val marketCode = ratePlan.marketType.get.marketCode<br>              val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = marketCode))<br>              rateplanupdated<br>            }<br>          }<br>          rec.copy(ratePlans = updatedRatePlans)<br>        }<br>        row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>      }<br>      updatedList<br>    } catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception while fetching from docdb: &quot; + ex.printStackTrace())<br>        List.empty[ReservationCollationEvent]<br>    }<br>  }<br><br><br><br>  2nd method==============================================================================<br>   def fetchPMSfromRateplan(<br>                              reservationRecordsList: List[ReservationCollationEvent]<br>                            )(implicit jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails):<br>    List[ReservationCollationEvent] = {<br><br>      val updatedList = reservationRecordsList. map { row =&gt;<br>        try {<br>          val updatedPayload = row.reservationPayload.HotelReservation.map { rec =&gt;<br><br>            val updatedRatePlans = rec.ratePlans.map { ratePlan =&gt;<br>              if (ratePlan.details.propertyMarketSegmentation.getOrElse(&quot;&quot;).isEmpty) {<br><br>                val documentDbDetails = connectionConfigDetails.documentconfig.get // -- 1. fetch credentials and create conf to connect to docdb<br>                val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails) // -- 2. connect to mongoclient and get database and collection -- need to check if code is connecting or not to mongo<br>                val mongodbClient = getMongoClient(conf, DocumentDBInput)<br><br>                val db = mongodbClient.getDatabase(jobConfigurationDetails.doucmentdbmap.get(&quot;documentdbname&quot;))<br>                val collection = db.getCollection(jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;))<br>                val key = &quot;RATEDFENDOC_&quot; + row.reservationPayload.HotelReservation.get.hotels.head.propertyCode.code + &quot;_&quot; + row.reservationPayload.HotelReservation.get.ratePlans.head.code + &quot;_&quot;<br>                val regexPattern = s&quot;&quot;&quot;^$key.*_$$&quot;&quot;&quot;.r.pattern<br>                val filter = Filters.regex(&quot;_id&quot;, regexPattern)<br>                val docdbdata = collection.find(filter).first().toJson() // -- data in form of Document fetched from docdb and parse it to get ratePlan.ratePlanDetails.marketType.marketCode<br>                val parsedocdata = JsonUtil.fromJson[RatesBucketData](docdbdata)<br>                val pmsdata = parsedocdata.ratePlanDetails.head.marketType.get.marketCode<br>                val rateplanupdated = ratePlan.copy(details = ratePlan.details.copy(propertyMarketSegmentation = pmsdata))<br>                rateplanupdated<br>              } else {<br><br>                ratePlan<br>              }<br>            }<br>            rec.copy(ratePlans = updatedRatePlans)<br>          }<br>          row.copy(reservationPayload = row.reservationPayload.copy(HotelReservation = updatedPayload))<br>        } catch {<br>          case exception: Exception =&gt;<br>            new ExceptionHandler(exception).handle(<br>              Map(<br>                jobIdentifiter -&gt; jobConfigurationDetails.jobidentifier,<br>                jobName -&gt; jobConfigurationDetails.jobname,<br>                customErrorMessage -&gt; {<br>                  s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records from<br>                     |${jobConfigurationDetails.doucmentdbmap.get(&quot;documentcollectionname&quot;)} for reservation ID ${row.reservationInfo}&quot;&quot;&quot;.stripMargin<br>                }<br>              )<br>            )<br>            ReservationCollationEvent(row.reservationInfo, row.reservationPayload, isRecoverable = true)<br>        }<br>      }<br><br>      updatedList<br>    }<br><br><br>    ================================================================================================<br><br><br>    wither take a flag true or flase (rateplan recoverable)<br><br>    or try catch<br><br><br><br><br><br><br><br>    ==============================================================================================<br><br>    mappartion and mapgroup difference:<br><br><br>    this time it went to the catch block of  getLatestReservationRecord as this (Exception in getLatestReservationRecord:) is printed in the logs  <br>catch {<br>      case ex: Exception =&gt;<br>        log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>        throw ex // rethrow the exception to be caught by reservationCollation<br>    }<br><br><br>but again it didnt go to the exception of <br><br><br>idealy if we get the exception in getLatestReservationRecord which is enclosed in thr try block of reservationCollation method it should have gone to catch block of  reservationCollation<br><br>what is causing the issue<br><br>def reservationCollation(rawReservationDS: Dataset[ReservationCollationEvent],<br>                           sparkSession: SparkSession)(implicit jobConfigDetails: JobConfigurationDetails,connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>    log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    //    TODO : ReservationEventsEntity<br>    sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>    //println(&quot;conf status:&quot;, sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;))<br>    log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    import sparkSession.implicits._<br><br>    val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>    log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>    implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>    /**<br>     * group the multiple records received in the micro batch and get the latest record<br>     */<br><br>    log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>      CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br>    try {<br>      val latestReservationRecordDS = rawReservationDS<br>        .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>        .mapGroups(getLatestReservationRecord)<br>      //println(&quot;5 Aug - Data after getLatestReservationRecord step - &quot;)<br>      //latestReservationRecordDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      val recoverableDocDBDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>          row.newNOldReservations.get.newReservation.nonEmpty ||<br>          row.newNOldReservations.get.reservationKey.nonEmpty<br>        )<br>        .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>      val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>      val validReservationRecordsDS = latestReservationRecordDS<br>        .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>        .map(row =&gt; row.newNOldReservations.get)<br>      //println(&quot;5 Aug - Data after filtration of getLatestReservationRecord step - &quot;)<br>      //validReservationRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>      /**<br>       * compare and modify the pay and stay details for reservation record and<br>       * add the same to each and every segment in the record<br>       */<br>      val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br>      //println(&quot;5 Aug - Data after mapping of validrecords step - &quot;)<br>      //newNOldRecordsDS.writeStream.format(&quot;console&quot;).outputMode(&quot;append&quot;).option(&quot;truncate&quot;, false).start()<br><br>      log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;,<br>        CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, customData))<br>      (newNOldRecordsDS, recoverableDocDBDF)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.info(&quot;entering in the catch block of reservationCollation &quot;)<br>//        new ExceptionHandler(exception).handle(<br>//          Map(<br>//            jobIdentifiter -&gt; jobConfigDetails.jobidentifier,<br>//            jobName -&gt; jobConfigDetails.jobname,<br>//            customErrorMessage -&gt; {<br>//              s&quot;&quot;&quot;fetchDocDBDocument :: Error Occurred while fetching records-2&quot;&quot;&quot;.stripMargin<br>//            }<br>//          )<br>//        )<br>        val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>        println(&quot;printing schema of  recoverableEvent&quot;)<br>        recoverableEvent.printSchema()<br>        val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>        finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>        println(&quot;checkpoint--------&quot;)<br>        produceRecoverableData(finalDF)<br>        val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>        val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>        (emptyDataset, emptyDataFrame)<br><br><br>    }<br>  }<br><br>Copy<br><br>To ensure that exceptions thrown in getLatestReservationRecord are caught by the reservationCollation method, several factors must be considered, especially with respect to how Spark handles exceptions in distributed computing contexts. One common issue is that Spark operations often execute in separate threads or executors, and exceptions may not propagate as expected to the driver program&#39;s try-catch blocks.<br><br>Here&#39;s a checklist and potential fixes to ensure exceptions propagate correctly:<br><br>Ensure Rethrown Exception:<br>Confirm that getLatestReservationRecord actually rethrows the exception after logging it.<br><br><br>Exception Handling in Spark Transformations:<br>When using Spark transformations (e.g., mapGroups), exceptions thrown in these transformations may not propagate back to the driver. Instead, they might be logged on the executor nodes.<br><br><br>Use mapPartitions for Better Exception Handling:<br>If you suspect the exception handling issue is related to the distributed nature of Spark, consider using mapPartitions to handle exceptions more granularly.<br><br><br>Checkpointing and Debugging:<br>Add additional logging to confirm the flow of execution and pinpoint where the exception might be getting swallowed or lost.<br>Here is a modified version of reservationCollation that includes more detailed logging and uses mapPartitions for better exception handling:<br><br>def reservationCollation(<br>    rawReservationDS: Dataset[ReservationCollationEvent],<br>    sparkSession: SparkSession<br>)(implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Spark conf is set with couchbase connection details&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups((key, records) =&gt; {<br>        try {<br>          getLatestReservationRecord(Some(key), records)<br>        } catch {<br>          case ex: Exception =&gt;<br>            log.error(&quot;Exception in mapGroups while processing key: &quot; + key, ex)<br>            throw ex<br>        }<br>      })<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;)<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;)<br>    (newNOldRecordsDS, recoverableDocDBDF)<br><br>  } catch {<br>    case exception: Exception =&gt;<br>      log.error(&quot;Exception in reservationCollation: &quot;, exception)<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      log.info(&quot;Printing schema of recoverableEvent&quot;)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      log.info(&quot;Checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>Key Changes and Additions:<br><br><br>Detailed Logging:<br>Added more logging to both the try and catch blocks to track the flow and pinpoint where the exception is being handled or missed.<br><br><br>Exception Handling in mapGroups:<br>Wrapped the logic within mapGroups in a try-catch block to catch and log exceptions specific to each group key.<br><br><br>Error Logging in reservationCollation Catch Block:<br>Ensured the exception is logged with a detailed message.<br>These changes should help in diagnosing the issue and ensure that exceptions are propagated correctly to the catch block of reservationCollation.<br><br><br>Copy<br>Thank you for your feedback.<br>Icon<br>============================================================================================================================<br><br><br>in case of throw use right and left in case of exceptions:<br><br><br><br><br>It seems that the exception is being logged in the mapGroups function, but it is not propagating up to the try block in reservationCollation. This behavior is typical in Spark, where exceptions occurring in distributed operations (like those inside mapGroups) do not always propagate back to the driver in a straightforward manner.<br><br>To handle exceptions in Spark operations more effectively, you can use a combination of techniques:<br><br>Collecting Results for Error Handling:<br>One approach is to collect the results of operations to the driver, where you can handle exceptions more reliably.<br><br><br>Using Accumulators or Broadcast Variables:<br>You can use Spark&#39;s accumulators or broadcast variables to signal errors back to the driver.<br><br><br>Custom Error Handling Logic:<br>Implement custom error handling within the distributed operations and handle as much as possible locally before propagating a summary error back to the driver.<br>Here&#39;s an example of how you might rework the mapGroups function to use a custom error-handling approach:<br><br>Modified getLatestReservationRecord with Error Handling:<br>def getLatestReservationRecord(<br>    reservationKey: Option[String],<br>    reservationRecords: Iterator[ReservationCollationEvent]<br>)(implicit conf: SparkConf, jobConfigurationDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): Either[Exception, NewNOldReservationRecordsErrorEntity] = {<br><br>  log.info(&quot;Fetching the latest reservation record out of list of reservation records with same reservation key - getLatestReservationRecord()&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  val reservationRecordsList = reservationRecords.toList<br><br>  try {<br>    val resRecWithCMS = fetchPMSfromRateplan(reservationRecordsList)<br>    Right(resRecWithCMS)<br>  } catch {<br>    case ex: Exception =&gt;<br>      log.error(&quot;Exception in getLatestReservationRecord: &quot; + ex.printStackTrace())<br>      Left(ex)<br>  }<br>}<br>Modified reservationCollation with Error Handling:<br>def reservationCollation(<br>    rawReservationDS: Dataset[ReservationCollationEvent],<br>    sparkSession: SparkSession<br>)(implicit jobConfigDetails: JobConfigurationDetails, connectionConfigDetails: ConnectionConfigDetails): (Dataset[ReservationEventsEntity], DataFrame) = {<br><br>  log.info(&quot;Starting the processing of Reservation Collation&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  sparkSession.conf.set(&quot;spark.sql.adaptive.enabled&quot;, false)<br>  log.info(&quot;conf status:&quot; + sparkSession.conf.get(&quot;spark.sql.adaptive.enabled&quot;),<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  import sparkSession.implicits._<br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br>  implicit val conf: SparkConf = getDocumentDbSparkConf(documentDbDetails)<br><br>  log.info(&quot;Starting the processing of fetching reservation key &amp; functioning to get latest reservation record based on that reservation key&quot;,<br>    CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, appNameForLogger, jobIdentifier, jobName, customData))<br><br>  try {<br>    val latestReservationRecordDS = rawReservationDS<br>      .groupByKey(reservationRecord =&gt; getReservationKey(reservationRecord, jobConfigDetails))<br>      .mapGroups((key, records) =&gt; {<br>        getLatestReservationRecord(Some(key), records) match {<br>          case Right(result) =&gt; result<br>          case Left(ex) =&gt;<br>            log.error(s&quot;Exception in mapGroups while processing key: $key&quot;, ex)<br>            throw ex<br>        }<br>      })<br><br>    val recoverableDocDBDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.get.oldReservation.nonEmpty ||<br>        row.newNOldReservations.get.newReservation.nonEmpty ||<br>        row.newNOldReservations.get.reservationKey.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get.newReservation)<br><br>    val recoverableDocDBDF = recoverableDocDBDS.toDF().select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br><br>    val validReservationRecordsDS = latestReservationRecordDS<br>      .filter(row =&gt; row.newNOldReservations.nonEmpty)<br>      .map(row =&gt; row.newNOldReservations.get)<br><br>    log.info(&quot;Comparing and modifying the pay and stay details for reservation record - reservationCollation()&quot;)<br><br>    val newNOldRecordsDS = validReservationRecordsDS.map(getReservationEvents_Map)<br><br>    log.info(&quot;Starting the process to write into DocumentDB &amp; Kafka - reservationCollation()&quot;)<br>    (newNOldRecordsDS, recoverableDocDBDF)<br><br>  } catch {<br>    case exception: Exception =&gt;<br>      log.error(&quot;Exception in reservationCollation: &quot;, exception)<br>      val recoverableEvent = rawReservationDS.map(x =&gt; x.reservationPayload.payLoad)<br>      log.info(&quot;Printing schema of recoverableEvent&quot;)<br>      recoverableEvent.printSchema()<br>      val finalDF = recoverableEvent.select(from_json(col(&quot;value&quot;), Encoders.product[HotelReservation].schema).as(&quot;value&quot;)).toDF()<br>      finalDF.writeStream.format(&quot;console&quot;).option(&quot;truncate&quot;, false).start()<br>      log.info(&quot;Checkpoint--------&quot;)<br>      produceRecoverableData(finalDF)<br>      val emptyDataset = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity])<br>      val emptyDataFrame = sparkSession.emptyDataset[ReservationEventsEntity](Encoders.product[ReservationEventsEntity]).toDF()<br><br>      (emptyDataset, emptyDataFrame)<br>  }<br>}<br>Explanation:<br><br><br>Returning Either from getLatestReservationRecord:<br>This method now returns an Either type, encapsulating either a successful result (Right) or an exception (Left).<br><br><br>Handling Either in mapGroups:<br>The mapGroups function processes the Either result, logging and rethrowing exceptions when they occur.<br><br><br>Logging and Exception Handling:<br>Added detailed logging to track where exceptions occur and ensure they are logged and rethrown properly.<br>With these changes, exceptions should be captured more effectively and propagate as expected to the reservationCollation method&#39;s catch block<br><br><br>===========================================================================<br><br><br>expalin these functions groupby key  or mapgroup or anything used wirth a very simple basic example shwoing how the transformation in happeing draw the tables as well  val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity]) <br><br> val dataGB: Dataset[GroupBlockEntity] = gbDataset.groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>      .mapGroups(transformGroupBlockRecords)<br>      .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br>ChatGPT<br>In Spark&#39;s DataFrame API, groupByKey, mapGroups, and flatMap are operations used for processing data in a distributed manner. Let&#39;s break down each step with a simple example and draw tables to illustrate the transformation.<br><br>Suppose we have a dataset of GroupBlockEntity objects where each object contains information about a group block. We want to group these entities by a specific key, perform some transformation on each group, and then flatten the resulting groups into a single dataset.<br><br>Here&#39;s a basic example:<br><br>Suppose we have the following dataset of GroupBlockEntity:<br><br>Row ID  Group ID  Data<br>1 A ...<br>2 B ...<br>1 C ...<br>3 A ...<br>2 B ...<br>We want to group these entities by the Row ID column.<br><br>groupByKey: This operation groups the data by the specified key (Row ID in our case).<br>After groupByKey operation, the data would be grouped like this:<br><br>Row ID  Group<br>1 [Entity1, Entity3]<br>2 [Entity2, Entity5]<br>3 [Entity4]<br><br><br>Reservation_key :  Dataset[(ReservationCollationEvent,String), <br>  Reservation_key   :               Dataset[(ReservationCollationEvent_1)]<br>mapGroups: This operation applies a function to each group. Let&#39;s say transformGroupBlockRecords is a function that takes a group and applies some transformation to it.<br>After applying transformGroupBlockRecords, the data might look like this:<br><br>Row ID  Transformed Group<br>1 [TransformedEntity1, TransformedEntity3]<br>2 [TransformedEntity2, TransformedEntity5]<br>3 [TransformedEntity4]<br>flatMap: This operation flattens the groups into a single dataset.<br>After flatMap, the dataset would be:<br><br>Transformed Entity<br>TransformedEntity1<br>TransformedEntity3<br>TransformedEntity2<br>TransformedEntity5<br>TransformedEntity4<br>So, combining all these operations together in Spark code, we have:<br><br>scala<br>Copy code<br>val dataGB: Dataset[GroupBlockEntity] = gbDataset<br>  .groupByKey(gkey =&gt; gkey.groupBlockPayload.groupBlockCUD.baseTable.rowId)<br>  .mapGroups(transformGroupBlockRecords)<br>  .flatMap(x =&gt; x)(ExpressionEncoder[GroupBlockEntity])<br>This code groups the dataset by rowId, applies the transformGroupBlockRecords function to each group, and then flattens the result into a single dataset.<br><br><br><br>example of the same<br>Suppose we have a dataset containing sales data, and we want to calculate the total revenue for each product. Each row represents a sale with the following schema:<br><br>Product ID  Quantity  Price<br>101 2 10<br>102 1 20<br>101 3 15<br>103 2 25<br>102 2 18<br>We want to group this data by Product ID, calculate the total revenue for each product (which is the sum of Quantity * Price for each sale), and then flatten the result.<br><br>Here&#39;s how you can achieve this in Spark:<br><br>scala<br>Copy code<br>// Define a case class for the sales data<br>case class Sale(productId: Int, quantity: Int, price: Int)<br><br>// Example data<br>val salesData = Seq(<br>  Sale(101, 2, 10),<br>  Sale(102, 1, 20),<br>  Sale(101, 3, 15),<br>  Sale(103, 2, 25),<br>  Sale(102, 2, 18)<br>)<br><br>// Create a DataFrame from the sales data<br>val salesDF = salesData.toDF<br><br>// Import necessary Spark functions<br>import org.apache.spark.sql.functions._<br><br>// Group by &#39;productId&#39;, calculate total revenue, and flatten the result<br>val totalRevenueDF = salesDF<br>  .groupByKey(_.getAs[Int](&quot;productId&quot;))<br>  .mapGroups { case (productId, iterator) =&gt;<br>    val totalRevenue = iterator.map { row =&gt;<br>      row.getAs[Int](&quot;quantity&quot;) * row.getAs[Int](&quot;price&quot;)<br>    }.sum<br>    (productId, totalRevenue)<br>  }<br>  .flatMap { case (productId, totalRevenue) =&gt;<br>    Iterator((productId, totalRevenue))<br>  }<br>  .toDF(&quot;productId&quot;, &quot;totalRevenue&quot;)<br><br>// Show the result<br>totalRevenueDF.show()<br>Output:<br><br>diff<br>Copy code<br>+---------+-------------+<br>|productId|totalRevenue|<br>+---------+-------------+<br>|      101|          65|<br>|      103|          50|<br>|      102|          76|<br>+---------+-------------+<br>In this example:<br><br>We define a case class Sale to represent the sales data.<br>We create a DataFrame salesDF from the example data.<br>We import necessary Spark functions.<br>We group the DataFrame by &#39;productId&#39;.<br>Inside mapGroups, we calculate the total revenue for each group.<br>Finally, we flatten the result using flatMap and convert it to a DataFrame totalRevenueDF.<br><br><br><br><br>===============================================================================</pre><p>price-plan code</p><pre>Pricelan_code:<br>priceplan_test<br><br><br>Pricelan_code:<br>entity<br>exceptions<br>helper<br>job<br>udfs<br>util<br><br><br><br><br><br><br>hepler<br><br>priceplanhelper<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.helper<br><br>.marriott.mgpapp.spark.priceplan.ingestion.util.BuilderUtil.prepareDocumentDBReaderWithFilter<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{Deserializer, InputType, JobType, Serializer}<br>import com.marriott.rm.platform.util.{ConsumerHelperBuilder, ProducerHelperBuilder, ReaderBuilder}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br>import org.apache.spark.sql.types._<br><br>import java.time.format.DateTimeFormatter<br>import java.time.{LocalDate, LocalDateTime}<br><br>trait PricePlanIngestionHelper extends Serializable with StreamingJob {<br><br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br>  val customMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, loggerAppName, jobIdentifier, jobName, map)<br><br>  val jobConfigurationDetails: JobConfigurationDetails<br><br>  def consumeDataFromEEH(inputType: InputType.Type)(implicit sparkSession: SparkSession): DataFrame = {<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    val startingOffset: String = &quot;latest&quot;<br><br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br>      readStreamData.consume().select(&quot;value&quot;)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data from kafka topic :: Exception:&quot; + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br><br>        sparkSession.emptyDataFrame<br>    }<br>  }<br><br>  def convertToEntity(inputStr: String)(implicit sparkSession: SparkSession): MergedPricePlan = {<br>    try {<br>      log.info(&quot;Converting to Entity : convertToEntity()&quot;, customMeta)<br>      MergedPricePlan(JsonUtil.fromJson[PricePlan](inputStr), inputStr)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Input event parsing error :: Exception :&quot; + exception.printStackTrace(), customMeta)<br>        new EventDeserException(&quot;400&quot;, exception).handle()<br>        MergedPricePlan(PricePlan(null, null, null), inputStr)<br>    }<br>  }<br><br>  def filterPropertyCode(inputDataset: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    log.info(&quot;Filtering property codes : filterPropertyCode()&quot;, customMeta)<br><br>    val column: Column = col(PricePlanPropertyCodeCode).alias(&quot;property-code&quot;)<br>    val (mgpPropertyDf, nonMgpPropertyDf) = FilterPropertyCodeUtil.filterPropertyCode(inputDataset.toDF(), column)<br><br>    // Directly convert DataFrame to Dataset without JSON conversion<br>    val mgpDS = mgpPropertyDf.as[MergedPricePlan]<br>    mgpDS<br>  }<br><br>  def mandatoryFieldValidation(inputDataset: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): (Dataset[MergedPricePlan], Dataset[MergedPricePlan]) = {<br>    log.info(&quot;Mandatory field validations : mandatoryFieldValidation()&quot;, customMeta)<br>    ///record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code<br>    val cleanDS = inputDataset.filter { rec =&gt;<br>      !(rec.pricePlan.propertyCode == null || rec.pricePlan.pricePlanDetails == null || rec.pricePlan.generationTime == null || rec.pricePlan.generationTime.isEmpty ||<br>        rec.pricePlan.propertyCode.code == null || rec.pricePlan.propertyCode.code.isEmpty || rec.pricePlan.propertyCode.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.code == null || rec.pricePlan.pricePlanDetails.code.isEmpty || rec.pricePlan.pricePlanDetails.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.isEmpty || rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.isEmpty) ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.exists(x =&gt; x.code.isEmpty || x.code == null || x.code == &quot;&quot;  || x.guestsByType.isEmpty))<br>        )<br>    }<br>    val errorDS = inputDataset.filter { rec =&gt;<br>      (rec.pricePlan.propertyCode == null || rec.pricePlan.pricePlanDetails == null || rec.pricePlan.generationTime == null || rec.pricePlan.generationTime.isEmpty ||<br>        rec.pricePlan.propertyCode.code == null || rec.pricePlan.propertyCode.code.isEmpty || rec.pricePlan.propertyCode.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.code == null || rec.pricePlan.pricePlanDetails.code.isEmpty || rec.pricePlan.pricePlanDetails.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.exists(x =&gt; x.code.isEmpty || x.code == null || x.code == &quot;&quot; || x.guestsByType.isEmpty)))<br>    }<br><br>    (cleanDS, errorDS)<br>  }<br><br>  def processPricePlanRecords(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    log.info(&quot;Started Processing the Records | processPricePlanRecords()&quot;, customMeta)<br>    val latestPricePlanDs = priceData<br>      .groupByKey(record =&gt; (record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>      .mapGroups { case (_, iterator) =&gt;<br>        val buffer = iterator.toBuffer<br>        val latestRecord = buffer.maxBy(_.pricePlan.generationTime)<br>        latestRecord<br>      }<br>    latestPricePlanDs<br>  }<br><br>  def chunkByInventory(pricePlanRecs: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    // Flatten the dataset<br>    try {<br>      val flattened = pricePlanRecs.flatMap { pricePlanElem =&gt;<br>        pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.flatMap { ratePeriod =&gt;<br>          ratePeriod.inventoryTypes.map { pricePoint =&gt;<br>            Flattened(pricePlanElem, ratePeriod, pricePoint)<br>          }<br>        }<br>      }<br><br>      // Group by inventoryTypeCode and reconstruct<br>      val grouped = flattened.groupBy(&quot;pricePlanMerged.pricePlan.propertyCode.code&quot;, &quot;pricePoint.code&quot;)<br>        .agg(<br>          collect_list(&quot;pricePlanMerged&quot;).as(&quot;pricePlansMerged&quot;),<br>          collect_list(struct(&quot;ratePeriod&quot;, &quot;pricePoint&quot;)).as(&quot;ratePeriodsAndPoints&quot;)<br>        ).as[(String, String, Seq[MergedPricePlan], Seq[(RatePeriod, PricePoint)])]<br><br>      // Reconstruct the PricePlan dataset<br>      val transformed = grouped.flatMap { case (propertyCode, inventoryTypeCode, pricePlansMerged, ratePeriodsAndPoints) =&gt;<br>        pricePlansMerged.distinct.map { pricePlanMerged =&gt;<br>          val updatedRatePeriods = ratePeriodsAndPoints.map { case (ratePeriod, pricePoint) =&gt;<br>            ratePeriod.copy(inventoryTypes = Seq(pricePoint.copy(code = inventoryTypeCode)))<br>          }.sortBy(_.timespan.get.start)<br>          //added sort by<br>          pricePlanMerged.copy(pricePlan = pricePlanMerged.pricePlan.copy(pricePlanDetails = pricePlanMerged.pricePlan.pricePlanDetails.copy(ratePeriods = updatedRatePeriods)))<br>        }<br>      }<br>      transformed}<br>      catch<br>      {<br>        case e: Exception =&gt;<br>          log.error(s&quot;An error occurred in chunkByInventory: ${e.getMessage}&quot;, customMeta)<br>          sparkSession.emptyDataset[MergedPricePlan]<br>      }<br><br>  }<br><br>  def generateKeyA(propertycode: String, pricePlanCode: String, inventoryTypeCode: String): String = {<br>    s&quot;$PriceDocPrefix$UnderScore$propertycode$UnderScore$pricePlanCode$UnderScore$inventoryTypeCode&quot;<br>  }<br><br>  def getNonDelDocDB(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[PricePlanDocumentDBRecord] = {<br>    import sparkSession.implicits._<br>    try{<br>    val collectionName = jobConfigDetails.doucmentdbmap.get.get(PricePlanCollection).get<br><br>    def generateKeyB(propertycode: String, pricePlanCode: String): String = {<br>      s&quot;$PriceDocPrefix$UnderScore$propertycode$UnderScore$pricePlanCode&quot;<br>    }<br><br>    val keys = priceData.map(record =&gt; generateKeyB(record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code)).dropDuplicates()<br>      .collect()<br>    log.info(&quot;fetching ALL records based on documentDB based on propertyCode PricePlanCode| getNonDelDocDB()&quot;, customMeta)<br><br><br>    val priceDS: Dataset[PricePlanDocumentDBRecord] = if (keys.isEmpty) {<br><br>      sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    } else {<br>      val readers =<br>        keys.map { key =&gt;<br><br>      val read=prepareDocumentDBReaderWithFilter(<br>        pricePlanReaderBuilder,<br>        collectionName,<br>        classOf[PricePlanDocumentDBRecord],<br>        Some(col(&quot;_id&quot;)<br>          .rlike(key)<br>        )<br>      )<br>          read<br>    }<br>      val dataFrames = readers.flatMap { reader =&gt;<br>        val df = reader.as[PricePlanDocumentDBRecord]<br>        if (!df.isEmpty) Some(df) else None<br>      }<br>      dataFrames.reduceOption(_ union _).getOrElse(sparkSession.emptyDataset[PricePlanDocumentDBRecord])<br>    }<br>    val todayDate = DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_2).format(LocalDateTime.now())<br><br>    // Collect all inventory codes from price1<br>    val priceInventoryCodes = priceData.flatMap(_.pricePlan.pricePlanDetails.ratePeriods.flatMap(_.inventoryTypes.map(_.code)))<br>      .collect()<br>      .toSet<br>    val expiresUpdated = priceDS.map { doc =&gt;<br>      val inventoryCode = doc.pricePlanDetails.ratePeriods.headOption.flatMap(_.inventoryTypes.headOption.map(_.code))<br><br>      if (inventoryCode.exists(code =&gt; !priceInventoryCodes.contains(code))) {<br>        doc.copy(expiresAt = todayDate)<br>      } else {<br>        doc<br>      }<br>    }<br>    expiresUpdated}<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in getNonDelDocDB: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    }<br>  }<br><br>  def processNonDeleteRecords(priceData: Dataset[MergedPricePlan], priceDS: Dataset[PricePlanDocumentDBRecord])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails)<br>  : DataFrame = {<br><br>    try {<br>      log.info(s&quot;Inside method processNonDeleteFlow for Create|Update flow&quot;, customMeta)<br>      val collectionName = jobConfigDetails.doucmentdbmap.get.get(PricePlanCollection).get<br><br><br>      val deletedInventoryRecords = priceDS.filter(doc =&gt; doc.expiresAt != null)<br><br><br>      val nonDeleteInventoryRecords = priceDS.filter(doc =&gt; doc.expiresAt == null)<br><br>      deletedInventoryRecords.count()<br>      nonDeleteInventoryRecords.count()<br><br>      if (nonDeleteInventoryRecords.isEmpty) {<br>        val chunkedData = splitDOW(priceData)<br>        val destinationDS = (entityForLatestRecord(chunkedData)).union(deletedInventoryRecords)<br><br>        log.info(&quot;PropertyCode &amp; pricePlanCode does not found ; insert new record in documentDB | processNonDeleteFlow()&quot;, customMeta)<br><br>        BuilderUtil.insertDocumentDBRecords(destinationDS.toDF(), collectionName)<br><br>      } else {<br><br>        val thirtyDayFilteredPrice = cleanupThirtyDayPeriodOne(priceData)<br>        if (!thirtyDayFilteredPrice.isEmpty) {<br>          val chunkedData = splitDOW(thirtyDayFilteredPrice)<br>          val updatedChunkData = entityForLatestRecord(chunkedData)<br>          val finalDestinationDS = updatedChunkData.union(deletedInventoryRecords)<br>          writeToS3(finalDestinationDS)<br>          BuilderUtil.insertDocumentDBRecords(finalDestinationDS.toDF(), collectionName)<br>          log.info(&quot; Insert new Instance of key | processNonDeleteFlow&quot;, customMeta)<br>        }<br>      }<br><br>      priceDS.toDF()<br>    }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in getPriceRecords: ${e.getMessage}&quot;, customMeta)<br>        singleToRecoverable(priceData)<br>        sparkSession.emptyDataFrame<br>    }<br><br>  }<br><br>  def processDeletePricePlanDBRecords(pricePlanDS: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): DataFrame = {<br>    try {<br>      import sparkSession.implicits._<br>      log.info(&quot;Processing Delete actionEnum for PricePlanRecords from PricePlan Collection | processDeletePricePlanDBRecords()&quot;, customMeta)<br><br>      val collectionName = jobConfigDetails.doucmentdbmap.get(PricePlanCollection)<br><br>      def generateKeys(propertyCode: String, pricePlanCode: String, inventoryTypeCode: String): String = {<br>        s&quot;$PriceDocPrefix$UnderScore$propertyCode$UnderScore$pricePlanCode$UnderScore$inventoryTypeCode&quot;<br>      }<br><br>      val keys = pricePlanDS.map(record =&gt; generateKeys(record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code)).dropDuplicates().collect()<br><br><br>      val priceDS: Dataset[PricePlanDocumentDBRecord] = if (keys.isEmpty) {<br><br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>      } else {<br><br>        val reader = prepareDocumentDBReaderWithFilter(<br>          pricePlanReaderBuilder,<br>          collectionName,<br>          classOf[PricePlanDocumentDBRecord],<br>          Some(col(&quot;_id&quot;)<br>            .isin(keys: _*))<br>        )<br><br>        reader.as[PricePlanDocumentDBRecord]<br>      }<br>      log.info(&quot;fetching records based on documentDB based on propertyCode,PricePlanCode and Inv Type Code | processDeleteRatePlanDBRecords()&quot;, customMeta)<br><br>      log.info(&quot; Writing old record to S3 | processDeleteRatePlanDBRecords&quot;, customMeta)<br>      writeToS3(priceDS)<br><br>      val currentDate = DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_2).format(LocalDateTime.now())<br>      val resultDFWithExpire = priceDS.withColumn(expiresAt, lit(currentDate))<br>      BuilderUtil.insertDocumentDBRecords(resultDFWithExpire, collectionName)<br>      log.info(&quot;Inserted the updated records with expiresAt into PricePlan Collection | processDeleteRatePlanDBRecords()&quot;, customMeta)<br><br>      resultDFWithExpire<br>    }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in processDeleteRatePlanDBRecords: ${e.getMessage}&quot;, customMeta)<br><br>        singleToRecoverable(pricePlanDS)<br>        sparkSession.emptyDataFrame<br>    }<br>  }<br><br>  def splitDOW(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    try{<br>    def dowToIndices(dow: Seq[String]): Set[Int] = {<br>      val dowMap = Map(&quot;Monday&quot; -&gt; 1, &quot;Tuesday&quot; -&gt; 2, &quot;Wednesday&quot; -&gt; 3, &quot;Thursday&quot; -&gt; 4, &quot;Friday&quot; -&gt; 5, &quot;Saturday&quot; -&gt; 6, &quot;Sunday&quot; -&gt; 7)<br>      dow.map(d =&gt; dowMap(d)).toSet<br>    }<br><br>    // Function to expand date range based on DOW<br>    def expandDateRange(start: String, end: String, dow: Set[Int]): Seq[(String, String)] = {<br><br>      val startDate = java.time.LocalDate.parse(start)<br><br>      //var endDate = java.time.LocalDate.parse(end)<br>      /* val endDate = Option(end) match {<br>         case Some(&quot;&quot;) =&gt; startDate.plusDays(30)<br>         case Some(endValue) =&gt; LocalDate.parse(endValue)<br>         case None =&gt; startDate.plusDays(30)<br>       }*/<br>      val endDate = Option(end).filter(_.nonEmpty).map(LocalDate.parse).getOrElse(startDate.plusYears(4))<br>      var currentStart = startDate<br>      var currentDOW = currentStart.getDayOfWeek.getValue<br>      val expandedRanges = scala.collection.mutable.ListBuffer.empty[(String, String)]<br>      while (!currentStart.isAfter(endDate)) {<br>        if (dow.contains(currentDOW)) {<br>          var currentEnd = currentStart<br>          while (!currentEnd.isAfter(endDate) &amp;&amp; dow.contains(currentEnd.getDayOfWeek.getValue)) {<br>            currentEnd = currentEnd.plusDays(1)<br>          }<br>          expandedRanges += ((currentStart.toString, currentEnd.minusDays(1).toString))<br>          currentStart = currentEnd<br>        } else {<br>          currentStart = currentStart.plusDays(1)<br>        }<br>        currentDOW = currentStart.getDayOfWeek.getValue<br>      }<br>      expandedRanges<br>    }<br><br>    //UDF<br>    val udfReg = udf((start: String, end: String, dow: Seq[String]) =&gt;<br>      expandDateRange(start, end, dowToIndices(dow))<br>    )<br>    val priceDataWithExpandedDates = priceData.flatMap(row =&gt; {<br>      val ratePeriods = row.pricePlan.pricePlanDetails.ratePeriods<br>      ratePeriods.flatMap(ratePeriod =&gt; {<br>        val start = ratePeriod.timespan.flatMap(_.start).getOrElse(&quot;&quot;)<br>        val end = ratePeriod.timespan.flatMap(_.end).getOrElse(&quot;&quot;)<br>        val dow = ratePeriod.activeDowEnum<br>        val dowIndices = dowToIndices(dow)<br>        val expandedRanges = expandDateRange(start, end, dowIndices)<br>        expandedRanges.map { case (newStart, newEnd) =&gt;<br>          val updatedTimespan = Timespan(Some(newStart), Some(newEnd), ratePeriod.timespan.flatMap(_.duration))<br>          val updatedRatePeriod = ratePeriod.copy(timespan = Some(updatedTimespan))<br>          row.copy(pricePlan = row.pricePlan.copy(pricePlanDetails = row.pricePlan.pricePlanDetails.copy(ratePeriods = Seq(updatedRatePeriod))))<br>        }<br>      })<br>    })<br><br>    val explodedDF = priceDataWithExpandedDates<br>      .withColumn(ratePeriod, explode(col(&quot;pricePlan.pricePlanDetails.ratePeriods&quot;)))<br>      .withColumn(inventoryTypeSeq, explode(col(&quot;ratePeriod.inventoryTypes&quot;)))<br>      .withColumn(inventoryType, col(&quot;inventoryTypeSeq.code&quot;))<br><br>    val groupedDF = explodedDF<br>      .groupBy(<br>        col(pricePlan_propertyCode_code).as(propertyCode),<br>        col(pricePlan_pricePlanDetails_code).as(pricePlanCode),<br>        col(&quot;inventoryType&quot;)<br>      )<br>      .agg(<br>        collect_list(ratePeriod).as(ratePeriods),<br>        first(pricePlan).as(pricePlan),<br>        first(sourceString).as(sourceString)<br>      )<br>      .withColumn(pricePlan, struct(<br>        col(pricePlan_propertyCode).as(propertyCode),<br>        col(pricePlan_generationTime).as(generationTime),<br>        struct(<br>          col(pricePlan_pricePlanDetails_code).as(code),<br>          col(pricePlan_pricePlanDetails_actionEnum).as(actionEnum),<br>          col(pricePlan_pricePlanDetails_statusEnum).as(statusEnum),<br>          col(pricePlan_pricePlanDetails_priceGridDetails).as(priceGridDetails),<br>          col(pricePlan_pricePlanDetails_yieldManagement).as(yieldManagement),<br>          col(ratePeriods).as(ratePeriods),<br>          col(pricePlan_pricePlanDetails_priceGridStayRestriction).as(priceGridStayRestriction),<br>          col(pricePlan_pricePlanDetails_referencePriceGridCode).as(referencePriceGridCode),<br>          col(pricePlan_pricePlanDetails_pricing).as(pricing),<br>          col(pricePlan_pricePlanDetails_classificationsEnum).as(classificationsEnum),<br>          col(pricePlan_pricePlanDetails_currencyCode).as(currencyCode)<br>        ).as(pricePlanDetails)<br>      ))<br><br><br>    groupedDF.select(<br>      col(pricePlan),<br>      col(sourceString)<br>    ).as[MergedPricePlan]<br><br><br>  }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in splitDOW: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[MergedPricePlan]<br>    }}<br><br>  def comparePriceDatasets(merged: Dataset[MergedPricePlan], documentDb: Dataset[PricePlanDocumentDBRecord])(implicit spark: SparkSession): Dataset[PriceComparisonResult] = {<br><br>    import spark.implicits._<br>    log<br>      .info(&quot;Comparing PricePlan dataset with documentDB PricePlan Records | comparePriceDatasets()&quot;,<br>        customMeta<br>      )<br>    val joined = merged.joinWith(documentDb, merged(&quot;pricePlan.propertyCode.code&quot;) === documentDb(&quot;pricePlanDetails.propertyCode&quot;), joinType = &quot;left&quot;)<br>    val comparisonResult = joined.map { case (price, pricePlanDocumentDBRecord) =&gt; {<br><br>      val guestsByTypeComparison = (price.pricePlan.pricePlanDetails.priceGridDetails.get.freeGuestsByType == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.freeGuestsByType)<br><br>      val ratePeriodsComparison = price.pricePlan.pricePlanDetails.ratePeriods.zip(pricePlanDocumentDBRecord.pricePlanDetails.ratePeriods).forall {<br>        case (ratePeriod, ratePeriodDB) =&gt;<br>          ratePeriod.timespan == ratePeriodDB.timespan &amp;&amp;<br>            ratePeriod.activeDowEnum == ratePeriodDB.activeDowEnum &amp;&amp;<br>            ratePeriod.rounding == ratePeriodDB.rounding &amp;&amp;<br>            ratePeriod.amountAdd.get.value / math.pow(10, ratePeriod.amountAdd.get.numberOfDecimals.get.toInt) == ratePeriodDB.amountAdd.get.value &amp;&amp;<br>            ratePeriod.amountOff.get.value / math.pow(10, ratePeriod.amountOff.get.numberOfDecimals.get.toInt) == ratePeriodDB.amountOff.get.value &amp;&amp;<br>            ratePeriod.percentAdd == ratePeriodDB.percentAdd &amp;&amp;<br>            ratePeriod.percentOff == ratePeriodDB.percentOff &amp;&amp;<br>            ratePeriod.localAmountAdd.get.value / math.pow(10, ratePeriod.localAmountAdd.get.numberOfDecimals.get.toInt) == ratePeriodDB.localAmountAdd.get.value &amp;&amp;<br>            ratePeriod.localAmountOff.get.value / math.pow(10, ratePeriod.localAmountOff.get.numberOfDecimals.get.toInt) == ratePeriodDB.localAmountOff.get.value &amp;&amp;<br>            ratePeriod.level == ratePeriodDB.level &amp;&amp;<br>            ratePeriod.occupancy == ratePeriodDB.occupancy &amp;&amp;<br>            ratePeriod.guestTypeEnum == ratePeriodDB.guestTypeEnum &amp;&amp;<br>            ratePeriod.inventoryTypes.zip(ratePeriodDB.inventoryTypes).forall {<br>              case (invType, invTypeDB) =&gt;<br>                invType.code == invTypeDB.code &amp;&amp;<br>                  invType.monetaryAmount.get.value / math.pow(10, invType.monetaryAmount.get.numberOfDecimals.get.toInt) == invTypeDB.monetaryAmount.get.value &amp;&amp;<br>                  invType.localMonetaryAmount.get.value / math.pow(10, invType.localMonetaryAmount.get.numberOfDecimals.get.toInt) == invTypeDB.localMonetaryAmount.get.value &amp;&amp;<br>                  invType.changedInd == invTypeDB.hasChanged &amp;&amp;<br>                  invType.extraPersonCharge.get.value / math.pow(10, invType.extraPersonCharge.get.numberOfDecimals.get.toInt) == invTypeDB.extraPersonCharge.get.value<br>            }<br><br><br>      }<br><br><br>      val equal = (price.pricePlan.pricePlanDetails.code == pricePlanDocumentDBRecord.pricePlanDetails.code &amp;&amp;<br>        price.pricePlan.propertyCode.code == pricePlanDocumentDBRecord.pricePlanDetails.propertyCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code == pricePlanDocumentDBRecord.pricePlanDetails.inventoryCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.actionEnum == pricePlanDocumentDBRecord.pricePlanDetails.actionEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.statusEnum == pricePlanDocumentDBRecord.pricePlanDetails.statusEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.name == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.name &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.typeEnum == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.typeEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.taxInclusionEnum == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.taxInclusionEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.priceEndsWith == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.priceEndsWith &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productCategoryCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productCategoryCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productClassCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productClassCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults &amp;&amp;<br>        guestsByTypeComparison &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.yieldChangedInd == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.yieldChangedInd &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.typeEnum == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.typeEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.flatAmount.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.percent == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.percent &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value &amp;&amp;<br>        ratePeriodsComparison &amp;&amp;<br>        price.pricePlan.pricePlanDetails.pricing == pricePlanDocumentDBRecord.pricePlanDetails.pricing &amp;&amp;<br>        price.pricePlan.pricePlanDetails.classificationsEnum == pricePlanDocumentDBRecord.pricePlanDetails.classificationsEnum<br><br>        )<br>      PriceComparisonResult(price, equal)<br>    }<br>    }<br>    log<br>      .info(&quot;Comparing PricePlan dataset with documentDB PricePlan Records Completed | comparePriceDatasets()&quot;,<br>        customMeta<br>      )<br>    comparisonResult<br>  }<br><br>  def entityForLatestRecord(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails)<br>  : Dataset[PricePlanDocumentDBRecord] = {<br>    import sparkSession.implicits._<br>    try{<br>    val destinationDS: Dataset[PricePlanDocumentDBRecord] = priceData.map { pricePlanElem =&gt;<br>      createPriceDocument(pricePlanElem, generateKeyA(pricePlanElem.pricePlan.propertyCode.code, pricePlanElem.pricePlan.pricePlanDetails.code, pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>    }<br>    destinationDS}<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in splitDOW: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    }<br><br>  }<br><br>  def createPriceDocument(price: MergedPricePlan, key: String): PricePlanDocumentDBRecord = {<br><br><br>    val ratePeriodElements = scala.collection.mutable.ListBuffer[RatePeriodDB]()<br><br>    price.pricePlan.pricePlanDetails.ratePeriods.foreach(rate =&gt; {<br>      val counts = rate.inventoryTypes.flatMap(_.guestsByType.flatMap(_.count))<br>      val minDoubleRateCount = counts.filter(_ &gt;= 2)<br>      val singleOccupancyRate = if (counts.nonEmpty) Some(counts.min) else None<br>      val doubleOccupancyRate = if (minDoubleRateCount.nonEmpty) Some(minDoubleRateCount.min) else if (counts.contains(1)) Some(1) else None<br>      val currencyCdPricePlanDetails = price.pricePlan.pricePlanDetails.currencyCode<br><br>      val singleOccInv = rate.inventoryTypes.filter(_.guestsByType.get.count == singleOccupancyRate).head<br>      val doubleOccInv = rate.inventoryTypes.filter(_.guestsByType.get.count == doubleOccupancyRate).head<br>      val currencyCode = if (singleOccInv.monetaryAmount.isDefined &amp;&amp; singleOccInv.monetaryAmount.get.currencyCode.isDefined) singleOccInv.monetaryAmount.get.currencyCode.get else currencyCdPricePlanDetails<br><br>      ratePeriodElements += RatePeriodDB(<br>        rate.activeDowEnum, rate.rounding,<br>        tieredRates = None,<br>        if (singleOccInv.monetaryAmount.isDefined) {<br>          if (singleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            singleOccInv.monetaryAmount.get.value / math.pow(10, singleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            singleOccInv.monetaryAmount.get.value<br><br>        } else null,<br><br>        if (singleOccInv.monetaryAmount.isDefined) {<br>          if (singleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            singleOccInv.monetaryAmount.get.value / math.pow(10, singleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            singleOccInv.monetaryAmount.get.value<br><br>        } else null,<br><br>        if (doubleOccInv.monetaryAmount.isDefined) {<br>          if (doubleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            doubleOccInv.monetaryAmount.get.value / math.pow(10, doubleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            doubleOccInv.monetaryAmount.get.value<br><br>        } else null<br>        ,<br>         <br>        if (doubleOccInv.monetaryAmount.isDefined) {<br>          if (doubleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            doubleOccInv.monetaryAmount.get.value / math.pow(10, doubleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            doubleOccInv.monetaryAmount.get.value<br><br>        } else null<br>        ,<br>        if (rate.amountAdd.isDefined) {<br>          if (rate.amountAdd.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.amountAdd.get.value / math.pow(10, rate.amountAdd.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.amountAdd.get.value))<br>        } else None,<br>        if (rate.amountOff.isDefined) {<br>          if (rate.amountOff.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.amountOff.get.value / math.pow(10, rate.amountOff.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.amountOff.get.value))<br>        } else None,<br>        rate.percentAdd,<br>        rate.percentOff,<br>        if (rate.localAmountAdd.isDefined) {<br>          if (rate.localAmountAdd.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.localAmountAdd.get.value / math.pow(10, rate.localAmountAdd.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.localAmountAdd.get.value))<br>        } else None,<br>        if (rate.localAmountOff.isDefined) {<br>          if (rate.localAmountOff.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.localAmountOff.get.value / math.pow(10, rate.localAmountOff.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.localAmountOff.get.value))<br>        } else None,<br>        rate.level,<br>        rate.occupancy,<br>        rate.guestTypeEnum,<br>        rate.inventoryTypes.map(<br>          invElem =&gt;<br>            PricePointDB(<br>              invElem.code,<br>              if (invElem.monetaryAmount.isDefined) {<br>                if (invElem.monetaryAmount.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.monetaryAmount.get.value / math.pow(10, invElem.monetaryAmount.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.monetaryAmount.get.value))<br>              } else None,<br>              if (invElem.localMonetaryAmount.isDefined) {<br>                if (invElem.localMonetaryAmount.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.localMonetaryAmount.get.value / math.pow(10, invElem.localMonetaryAmount.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.localMonetaryAmount.get.value))<br>              } else None,<br>              if (invElem.extraPersonCharge.isDefined) {<br>                if (invElem.extraPersonCharge.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.extraPersonCharge.get.value / math.pow(10, invElem.extraPersonCharge.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.extraPersonCharge.get.value))<br>              } else None,<br>              invElem.changedInd, rate.occupancy, rate.guestTypeEnum, invElem.guestsByType<br>            )<br>        ),<br>        rate.timespan,<br>        Option(currencyCode)<br>      )<br>    }<br>    )<br><br>    val pricePlanRecords = PricePlanDetailsDB(price.pricePlan.pricePlanDetails.code,<br>      price.pricePlan.propertyCode.code, price.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code,<br>      price.pricePlan.pricePlanDetails.actionEnum, price.pricePlan.pricePlanDetails.statusEnum, Some(<br>        PriceGridDetailsDB(<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.name, price.pricePlan.pricePlanDetails.priceGridDetails.get.typeEnum,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd,<br>          if (price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.taxInclusionEnum,<br>          //price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.currencyCode,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.flatMap(_.extraPersonCharge).map(_.currencyCode).getOrElse(null),<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.priceEndsWith, price.pricePlan.pricePlanDetails.priceGridDetails.get.productCategoryCode,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.productClassCode, price.pricePlan.pricePlanDetails.priceGridDetails.get.productCode,<br>          if (price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.freeGuestsByType<br><br>        )), Some(<br>        YieldManagementDB(<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.yieldChangedInd,<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.typeEnum,<br>          if (price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.percent,<br>          if (price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value))<br>          } else None<br>        )),<br>      ratePeriodElements, price.pricePlan.pricePlanDetails.priceGridStayRestriction,<br>      price.pricePlan.pricePlanDetails.referencePriceGridCode, price.pricePlan.pricePlanDetails.pricing,<br>      price.pricePlan.pricePlanDetails.classificationsEnum, price.pricePlan.pricePlanDetails.currencyCode<br>    )<br><br>    PricePlanDocumentDBRecord(key, pricePlanRecords, null)<br>  }<br><br>  val avroSchema = new StructType()<br>    .add(&quot;timeStamp&quot;, StringType, nullable = true)<br>    .add(&quot;metadata&quot;, MapType(StringType, StringType, valueContainsNull = true))<br>    .add(&quot;retryCounter&quot;, IntegerType, nullable = true)<br>    .add(&quot;payLoad&quot;, StringType, nullable = true)<br>    .add(&quot;eventVersion&quot;, StringType, nullable = true)<br>    .add(&quot;eventSource&quot;, StringType, nullable = true)<br>    .add(&quot;destinationTopic&quot;, StringType, nullable = true)<br>    .add(&quot;eventIdentifier&quot;, StringType, nullable = true)<br>    .add(&quot;payLoadType&quot;, StringType, nullable = true)<br>    .add(&quot;recordNum&quot;, StringType, nullable = true)<br>    .add(&quot;error&quot;, StringType, nullable = true)<br>    .add(&quot;errorType&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;message&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;warning&quot;, ArrayType(StringType, containsNull = true))<br><br><br>  def produceDataToEEH(data: DataFrame)(implicit checkpointPath: Option[String]): Unit = {<br><br>    val destinationTopic = KafkaProperties.destinationTopic<br>    val schemaPath = &quot;merge-schema.avsc&quot;<br><br>    try {<br><br>      val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(schemaPath))<br>        .withCaseClass(Some(classOf[EEHData]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>      log.info(&quot;Pushed into kafka topic&quot;, customMeta)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br><br>  def writeDsToKafka(genTime: DataFrame, delRecs: DataFrame, nonDelRecs: DataFrame, pricePlanRecs: DataFrame)(implicit sparkSession: SparkSession,<br>                                                                                                              jobConfigurationDetails: JobConfigurationDetails): Dataset[String] = {<br>    val destinationTopic = jobConfigurationDetails.kafkatopicmap.get(&quot;DestinationTopic&quot;)<br>    val combinedRecs = delRecs.union(nonDelRecs)<br>    log<br>      .info(&quot;Kafka Writing records | writeDsToKafka()&quot;,<br>        customMeta<br>      )<br><br>    val combinedWithGenTime = combinedRecs.join(<br>      genTime,<br>      combinedRecs(&quot;pricePlanDetails.propertyCode&quot;) === genTime(&quot;propertyCode&quot;) &amp;&amp;<br>        combinedRecs(&quot;pricePlanDetails.code&quot;) === genTime(&quot;pricePlanCode&quot;),<br>      &quot;left&quot;<br>    ).withColumn(&quot;generationTime&quot;, coalesce(genTime(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>    val initialPricePlanExploded = pricePlanRecs.withColumn(propertyCode, col(&quot;pricePlan.propertyCode.code&quot;))<br>      .withColumn(pricePlanId, col(&quot;pricePlan.pricePlanDetails.code&quot;))<br>      .withColumn(actionEnum, col(&quot;pricePlan.pricePlanDetails.actionEnum&quot;))<br>      .withColumn(modifiedTimestamp, col(&quot;pricePlan.generationTime&quot;))<br>      .withColumn(ratePeriod, explode(col(&quot;pricePlan.pricePlanDetails.ratePeriods&quot;)))<br>      .withColumn(inventoryTypeSeq, explode(col(&quot;ratePeriod.inventoryTypes&quot;)))<br>      .withColumn(inventoryType, col(&quot;inventoryTypeSeq.code&quot;))<br>      .filter(not(col(actionEnum) === NoChange))<br>      .withColumn(isActive, when(col(actionEnum) === Delete, &quot;false&quot;)<br>        .otherwise(when(col(actionEnum) === Remove, &quot;false&quot;)<br>          .otherwise(&quot;true&quot;)))<br>      .select(propertyCode, pricePlanId, actionEnum, modifiedTimestamp, inventoryType, isActive)<br><br>    val pricePlanProcessedExploded = combinedWithGenTime.withColumn(propertyCode, col(&quot;pricePlanDetails.propertyCode&quot;))<br>      .withColumn(pricePlanId, col(&quot;pricePlanDetails.code&quot;))<br>      .withColumn(actionEnum, col(&quot;pricePlanDetails.actionEnum&quot;))<br>      .withColumn(modifiedTimestamp, col(&quot;generationTime&quot;))<br>      .withColumn(inventoryType, col(&quot;pricePlanDetails.inventoryCode&quot;))<br>      .filter(not(col(actionEnum) === NoChange)).dropDuplicates(&quot;propertyCode&quot;, &quot;pricePlanId&quot;, &quot;inventoryType&quot;)<br>      .withColumn(isActive, when(col(&quot;expiresAt&quot;).isNotNull, &quot;false&quot;).otherwise(&quot;true&quot;))<br>      .select(propertyCode, pricePlanId, actionEnum, modifiedTimestamp, inventoryType, isActive)<br>      .union(initialPricePlanExploded).dropDuplicates()<br><br>    val pricePlanExploded = pricePlanProcessedExploded<br>      .withColumn(&quot;mergerData&quot;,<br>        to_json(struct(propertyCode, pricePlanId, inventoryType, isActive, modifiedTimestamp)))<br>      .withColumn(EventSource, lit(jobName))<br>      .withColumn(EventVersion, lit(EmptyStr))<br>      .withColumn(EventIdentifier, lit(jobIdentifier))<br>      .withColumn(RetryCounter, lit(0))<br>      .withColumn(MetaData, typedLit(Map[String, String]()))<br>      .withColumn(&quot;DestinationTopic&quot;, lit(destinationTopic))<br>      .withColumn(RecordNum, lit(0))<br>      .withColumn(TimeStamp, current_timestamp())<br>      .withColumn(priceplan.PricePlanIngestion.Error, lit(EmptyStr))<br>      .withColumn(ErrorType, EmptyArray())<br>      .withColumn(Message, EmptyArray())<br>      .withColumn(Warning, EmptyArray())<br>      .withColumn(PayLoad, col(&quot;mergerData&quot;))<br>      .withColumn(PayLoadType, lit(Json))<br>      .withColumn(PricePlanEventJson,<br>        to_json(struct(TimeStamp, MetaData, RetryCounter, PayLoad, EventVersion, EventSource, &quot;DestinationTopic&quot;, EventIdentifier, PayLoadType,<br>          RecordNum, PricePlanIngestion.Error, ErrorType, Message, Warning)))<br>    import sparkSession.implicits._<br>    pricePlanExploded.select(&quot;PricePlanEventJson&quot;).write.format(&quot;console&quot;)<br>    pricePlanExploded.select(&quot;PricePlanEventJson&quot;).as[String]<br><br>  }<br><br><br>  def writeToS3(PricePlan: Dataset[PricePlanDocumentDBRecord])(implicit sparkSession: SparkSession, connectionConfigDetails: ConnectionConfigDetails, jobConfigDetails: JobConfigurationDetails): Boolean = {<br>    try {<br><br>      if (PricePlan.isEmpty) {<br>        log.info(&quot;DataFrame is empty. Not writing to S3.&quot;)<br>        return false<br>      }<br><br>      val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br><br>      log.info(&quot; inside writetos3 &quot;, customMeta)<br><br>      val S3PricePlanPath = S3Util.getCheckPointDirectory(s3bucket, jobName,<br>        jobConfigDetails.checkpointdirectory.get(pricePlanAuditBucket) + &quot;/&quot; + jobConfigDetails.checkpointdirectory.get(PricePlanS3backup))<br><br>      val S3FilePrefix = PricePlan.select(col(&quot;_id&quot;)).first().getString(0)<br>      val currentTimestamp = LocalDateTime.now.format(DateTimeFormatter.ofPattern(&quot;dd-MM-yyyy_HH_mm_ss&quot;))<br>      val S3PricePlanFilename = s&quot;$S3FilePrefix$UnderScore$currentTimestamp&quot;<br>      val todayDate = LocalDateTime.now.format(DateTimeFormatter.ofPattern(&quot;yyyyMMdd&quot;))<br>      val S3DayFolder = s&quot;$todayDate&quot;<br>      val s3Path = s&quot;$S3PricePlanPath/$S3DayFolder/$S3PricePlanFilename&quot;<br>      println(&quot;s3 path :&quot;, s3Path)<br><br>      PricePlan.withColumn(&quot;date&quot;, lit(S3DayFolder)).withColumn(&quot;propertyCode&quot;, col(&quot;pricePlanDetails.propertyCode&quot;))<br>        .write<br>        .format(&quot;json&quot;)<br>        .mode(&quot;append&quot;)<br>        .partitionBy(&quot;propertyCode&quot;, &quot;date&quot;)<br>        .save(s3Path)<br><br>      log.info(&quot;Written To S3&quot;,<br>        customMeta)<br>      false<br>    }<br>    catch {<br>      case exception: Exception =&gt; log.error(&quot;Error while writing to S3 in pricePlan :: exception :&quot; + exception.printStackTrace(),<br>        customMeta)<br>        new S3BucketException(exception,<br>          customMeta).handle()<br>        true<br>    }<br>  }<br><br><br>  def cleanupThirtyDayPeriodOne(pricePlan: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br><br>    val currentDate = LocalDate.now()<br>    val thirtyDayPeriod = currentDate.minusDays(30)<br><br>    log.info(&quot;inside cleanupThirtyDayPeriod process&quot;, customMeta)<br><br>    try {<br>      val pricePlansWithCondition = pricePlan.map { plan =&gt;<br>        val hasValidEndDate = plan.pricePlan.pricePlanDetails.ratePeriods.exists { ratePeriod =&gt;<br>          val endDate = ratePeriod.timespan.get.end match {<br>            case Some(date) =&gt; LocalDate.parse(date)<br>            case None =&gt; LocalDate.parse(ratePeriod.timespan.get.start.get).plusYears(4)<br>          }<br>          endDate.isAfter(thirtyDayPeriod)<br>        }<br>        (plan, hasValidEndDate)<br>      }<br><br>      val excludedPlans = pricePlansWithCondition.filter(!_._2).collect()<br><br><br>      val filteredPricePlans = pricePlansWithCondition.filter(_._2).map(_._1)<br><br><br>      log.info(&quot;cleanupThirtyDayPeriod process completed &quot;, customMeta)<br>      filteredPricePlans<br><br>    } catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in cleanupThirtyDayPeriod: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[MergedPricePlan]<br>    }<br>  }<br><br>  def writeIrrErrorToKafka(errorDS: Dataset[MergedPricePlan], topic: String, checkpoint: String): Unit = {<br>    try {<br>      log.info(&quot;Writing irrecoverable error to kafka : writeErrorToKafka()&quot;, customMeta)<br>      val checkPointDirectory = Some(checkpoint)<br>      val irrecError = errorDS.withColumn(&quot;errorCode&quot;, lit(&quot;400&quot;))<br>        .withColumn(&quot;exception&quot;, lit(&quot;exception due to Irrecoverable error&quot;))<br>        .withColumn(&quot;errorMessage&quot;, lit(&quot;error due to incoming message&quot;))<br>        .withColumn(&quot;mainTopicMessage&quot;, col(&quot;sourceString&quot;))<br>        .withColumn(&quot;error&quot;,<br>          struct(&quot;errorCode&quot;, &quot;exception&quot;, &quot;errorMessage&quot;, &quot;mainTopicMessage&quot;))<br>        .select(&quot;error&quot;)<br><br>      val errorFinal = irrecError.select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br>        .withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br><br>      val schemaPath = &quot;irrecschema.avsc&quot;<br>      val producerHelper = new ProducerHelperBuilder(errorFinal, topic, jobTypes = JobType.Streaming)<br>        .withFilePath(Some(schemaPath))<br>        .withCheckPointPath(checkPointDirectory)<br>        .withCaseClass(Some(classOf[DLTError]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br><br>  def singleToRecoverable(inputDS: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Unit = {<br>    try {<br>      val finalDS = inputDS<br><br>        .select(col(&quot;sourceString&quot;)).coalesce(5).select(from_json(col(&quot;sourceString&quot;), Encoders.product[com.marriott.mmf.PricePlan].schema).as(&quot;value&quot;))<br><br><br>      val finalDF = finalDS.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br><br><br>      val producerHelper = new ProducerHelperBuilder(finalDF, KafkaProperties.recoverableErrorTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(&quot;recschema.avsc&quot;))<br>        .withCaseClass(Some(classOf[com.marriott.mmf.PricePlan]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br>}<br><br><br>jobs<br><br>main.scala<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.job<br><br>import com.marriott.elevate.spark.framework.jobs.MgpApp<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.loggerAppName<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.CustomloggerMetadata<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.SparkConf<br><br>object Main extends MgpApp {<br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  val customMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;,loggerAppName, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, map)<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br><br>  val pricePlanIngestionJob = new PricePlanIngestionJob(jobConfigurationDetails = jobConfigDetails,<br>    connectionConfigDetails = connectionConfigDetails,<br>    jobName = jobConfigDetails.jobname,<br>    jobIdentifier = jobConfigDetails.jobidentifier,<br>    triggerInterval = jobConfigDetails.triggerinterval.get)<br>  override def getSparkSession: Option[SparkConf] = Some(new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;PricePlanIngestion&quot;).setMaster(&quot;yarn&quot;))<br>  pricePlanIngestionJob.start()<br><br>}<br><br><br>priceplaningestionjob:<br><br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.job<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.defaultBucket<br>import com.marriott.elevate.spark.framework.traits.StreamingJob<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion._<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity._<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper.PricePlanIngestionHelper<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{InputType, OperationMode}<br>import com.marriott.rm.platform.util.ReaderBuilder<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql.functions.{col, from_json, lit}<br>import org.apache.spark.sql.streaming.Trigger<br>import org.apache.spark.sql.{Dataset, SparkSession}<br><br><br>class PricePlanIngestionJob(val jobConfigurationDetails: JobConfigurationDetails,<br>                            val connectionConfigDetails: ConnectionConfigDetails,<br>                            override val jobName: String,<br>                            override val jobIdentifier: String,<br>                            override val triggerInterval: Int)(implicit sparkSession: SparkSession)<br>  extends PricePlanIngestionHelper<br>    with StreamingJob<br>    with Serializable {<br><br>  this: PricePlanIngestionHelper =&gt;<br>  override val customMeta = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, loggerAppName, jobIdentifier, jobName, map)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val jobConfig: JobConfigurationDetails = jobConfigurationDetails<br><br><br>  val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)<br>  implicit val checkpointDirMap: Map[String, String] = jobConfigurationDetails.checkpointdirectory.get<br>  implicit val checkpointPath: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkpointDirMap.get(PricePlanIngestionCheckPoint).get))<br><br><br>  override def process(args: Map[String, String]): Unit = {<br>    /**<br>     * Consuming EEH Kafka topic data<br>     * Using UDF method, File is not required<br>     * */<br>    val data = consumeDataFromEEH(InputType.Avro)<br>    import sparkSession.implicits._<br>    val PricePlanDs = data.map(x =&gt; convertToEntity(x.getString(0)))<br>    val filteredDs = filterPropertyCode(PricePlanDs)<br><br>    val mandatoryFieldCheck = mandatoryFieldValidation(filteredDs)<br><br>    log.info(&quot;Mandatory attribute check has completed&quot;, customMeta)<br><br>    mandatoryFieldCheck._1.writeStream.foreachBatch { (batchDs: Dataset[MergedPricePlan], batchId: Long) =&gt;<br>      implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br><br>      val pricePlanInitial = processPricePlanRecords(batchDs)<br>      val pricePlanRecs = chunkByInventory(pricePlanInitial)<br>      val splitDOWRecs = splitDOW(pricePlanRecs)<br>      val genTime = pricePlanRecs.select(<br>        $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>        $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>        $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;)<br>      ).dropDuplicates(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;).toDF()<br><br>      val deleteFlow = pricePlanRecs.filter(_.pricePlan.pricePlanDetails.actionEnum.get == Delete)<br>      val delRecs = processDeletePricePlanDBRecords(deleteFlow)<br><br>      val nonDeleteFlow = pricePlanRecs.filter(_.pricePlan.pricePlanDetails.actionEnum.get != Delete)<br>      val initialNonDelRecs = getNonDelDocDB(nonDeleteFlow).persist()<br>      initialNonDelRecs.foreach(_ =&gt; ())<br>      processNonDeleteRecords(nonDeleteFlow, initialNonDelRecs)<br><br>      //here logic change required as non del flow is also having del,so use flag<br>      val flattenedJsonRecordsDataset = writeDsToKafka(genTime, delRecs, initialNonDelRecs.toDF(), splitDOWRecs.toDF())<br>      val jsonDF = flattenedJsonRecordsDataset.select(from_json(col(&quot;PricePlanEventJson&quot;), avroSchema).alias(&quot;value&quot;))<br>      val dataDF = jsonDF.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br>      produceDataToEEH(dataDF)<br>      initialNonDelRecs.unpersist()<br>      (): Unit<br>    }<br>      .option(&quot;checkpointLocation&quot;, checkpointPath.get)<br>      .trigger(Trigger.ProcessingTime(triggerInterval))<br>      .start()<br><br>    val errorDS = PricePlanDs.filter(x =&gt; x.pricePlan.pricePlanDetails == null)<br><br>    writeIrrErrorToKafka(errorDS.union(mandatoryFieldCheck._2), KafkaProperties.irrecoverableErrorTopic, S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(defaultBucket), jobName,<br>      jobConfigurationDetails.checkpointdirectory.get(IrrecoverableErrorCheckpointDir)))<br><br><br>    sparkSession.streams.awaitAnyTermination<br><br>  }<br><br><br>}<br><br>util<br><br>udfs:<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.udfs<br><br>import java.sql.Date<br>import java.text.SimpleDateFormat<br>import java.util.Calendar<br>import org.apache.spark.sql.functions.udf<br><br>object SparkUdfs {<br>  lazy val EmptyArray = udf(() =&gt; Array.empty[Int])<br><br>  }<br><br><br><br><br>builderUtil.scala<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.util<br><br><br>import com.marriott.rm.platform.enums.OperationMode<br>import com.marriott.rm.platform.util.{MongoClientBuilder, MongoClientConnection, Reader, ReaderBuilder, WriterBuilder}<br>import com.mongodb.client.MongoClient<br>import com.mongodb.spark.sql.connector.config.WriteConfig.OperationType<br>import org.apache.spark.sql.functions.col<br>import org.apache.spark.sql.{Column, SparkSession}<br><br>import scala.reflect.runtime.universe.TypeTag<br>import org.apache.spark.sql.{Column, DataFrame, SparkSession}<br><br>object BuilderUtil {<br><br>  private var mongoClient: MongoClient = null<br><br>  /** Prepare and Get the Mongo Client<br>   *<br>   * @return - MongoClient<br>   */<br>  def getMongoClient(connection :MongoClientConnection = new MongoClientBuilder().build()): MongoClient = {<br>    if (mongoClient == null)<br>      mongoClient = connection.getMongoClient<br>    mongoClient<br>  }<br><br>  /** Prepare Document DB Reader With Filter for batch DS<br>   *<br>   * @param collectionName  - collection name<br>   * @param filter          - filter<br>   * @param collectionClass - case class for mapping<br>   * @param sparkSession    - Spark Session<br>   * @tparam T - Type of the Case Class<br>   * @return<br>   */<br>  def prepareDocumentDBReaderWithFilter[T &lt;: Product : TypeTag](<br>                                                                 readerBuilder: ReaderBuilder[T],<br>                                                                 collectionName: String,<br>                                                                 collectionClass: Class[T],<br>                                                                 filter: Option[Column] = None<br>                                                               )(implicit<br>                                                                 sparkSession: SparkSession<br>                                                               ): DataFrame = {<br><br>    val reader = readerBuilder<br>      .withCollection(collectionName)<br>      .withClass(collectionClass)<br><br>    if (filter.isDefined)<br>      reader<br>        .withApplyFilterColumn(true)<br>        .withFilterColumn(filter.get<br>          &amp;&amp; col(&quot;expiresAt&quot;).isNull<br>        )<br>        .build().read()<br>    else reader.build().read()<br>  }<br><br>  def insertDocumentDBRecords[T &lt;: Product : TypeTag](dataFrame: DataFrame,<br>                                                      collectionName: String,<br>                                                      operationType: OperationType = OperationType.REPLACE,<br>                                                      mode: OperationMode.Type = OperationMode.Batch<br>                                                     ) = {<br>    val writerBuilder = new WriterBuilder(dataFrame, mode)<br>      .withCollection(collectionName)<br>      .withOperationType(operationType)<br>    writerBuilder.build().write()<br>  }<br><br>}<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>priceplan_test<br><br><br><br><br>package com.marriott.mgpapp.spark.priceplan.test.helper<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{IrrecoverableErrorTopic, defaultBucket}<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.IrrecoverableErrorCheckpointDir<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.{CurrValue, GuestsByType, MergedPricePlan, PriceComparisonResult, PriceGridDetailsDB, PricePlan, PricePlanDetails, PricePlanDetailsDB, PricePlanDocumentDBRecord, PricePointDB, Pricing, Range, RatePeriodDB, StayRestriction, TieredRate, Timespan, YieldManagementDB}<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper.PricePlanIngestionHelper<br>import com.marriott.mgpapp.spark.priceplan.ingestion.util.BuilderUtil.prepareDocumentDBReaderWithFilter<br>import com.marriott.mmf.{AmountRule, ClassificationEnum, CurrencyAmount, DependencyAttribute, Description, DowEnum, LOSPriceGrid, LOSPricing, LOSPricingActionEnum, Locale, MarketType, PriceGridTypeEnum, PriceRange, PricingFrequencyEnum, PropertyChain, PropertyCode, ProtectionLock, RatePeriod, RatePlanActionEnum, RoomBucket, SourceCRSEnum, TaxDetails, TaxInclusionEnum, TaxItem, TaxOverrideScope, TaxPeriod, TaxScope, TaxValue}<br>import com.marriott.rm.platform.enums.{InputType, OperationMode}<br>import com.marriott.rm.platform.util.{Reader, ReaderBuilder}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.apache.spark.sql.execution.streaming.MemoryStream<br>import org.apache.spark.sql.functions.{coalesce, col, from_json, lit}<br>import org.apache.spark.sql.types.StringType<br>import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession}<br>import org.mockito.ArgumentMatchers<br>import org.mockito.ArgumentMatchers.anyString<br>import org.mockito.Mockito.lenient<br>import org.mockito.scalatest.MockitoSugar<br>import org.scalatest.BeforeAndAfterAll<br>import org.scalatest.funsuite.AnyFunSuite<br>import org.scalatest.matchers.should.Matchers<br><br>import scala.reflect.runtime.universe.TypeTag<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper<br>case class MyClass(testColumn: String)<br>class PricePlanIngestionHelperTest extends AnyFunSuite<br>  with PricePlanIngestionHelper<br>  with TestHelper<br>  with java.io.Serializable<br>  with BeforeAndAfterAll<br>  with Matchers<br>  with MockitoSugar<br>{<br>  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(&quot;hduser&quot;))<br>  var sparkSession1: SparkSession = _<br>  var loadedDataFrame: DataFrame = _<br>  var inputDF: DataFrame = _<br><br><br>  override protected def beforeAll() {<br><br>    sparkSession1 = SparkSession<br>            /*.builder()<br>      .master(&quot;local[*]&quot;)<br>      .appName(&quot;Tests&quot;)<br>      .getOrCreate()*/<br>      .builder()<br>      .appName(&quot;PricePlanTest&quot;)<br>      .master(&quot;local&quot;)<br>      .config(&quot;spark.io.compression.codec&quot;, &quot;snappy&quot;)<br>      .getOrCreate()<br><br>  }<br><br>  override protected def afterAll {<br>    sparkSession1.stop()<br>  }<br><br>  def setupMockReaderBuilder[T &lt;: Product : TypeTag](expectedData: Seq[T]): ReaderBuilder[T] = {<br>    val mockReaderBuilder = mock[ReaderBuilder[T]]<br>    val mockReader = mock[Reader[T]]<br>    val expectedDF = sparkSession1.createDataFrame(expectedData)<br><br>    lenient().when(mockReaderBuilder.withCollection(anyString())).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withClass(any)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withApplyFilterColumn(anyBoolean)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withFilterColumn(any)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.build()).thenReturn(mockReader)<br>    lenient().when(mockReader.read()).thenReturn(expectedDF)<br><br>    mockReaderBuilder<br>  }<br><br>  def samplePricePlanDocumentDBData(_id: String, code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10), Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Some(Seq(tieredRate)), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-08-10&quot;), None)), Some(&quot;USD&quot;))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetailsDB(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue))), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;))), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }<br> /* def samplePricePlanMergedData(_id: String,code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10),Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Seq(tieredRate), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-08-10&quot;), None)))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetails(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2)))) ), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue)) ), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;)) ), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;)) ), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }*/<br><br>  def newSamplePricePlanDocumentDBData(_id: String,code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10),Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Some(Seq(tieredRate)), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-05-30&quot;), None)), Some(&quot;USD&quot;))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetailsDB(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2)))) ), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue)) ), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;)) ), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;)) ), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }<br><br>  test(&quot;writeToS3 should write Dataset to S3&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    val pricePlanDS = pricePlanDetailsData.toDS()<br><br>    implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(jobname = &quot;PricePlanIngestion&quot;, jobidentifier = &quot;PricePlan&quot;, triggerinterval = Some(1000), kafkamaxrequestsize = Some(20971520), checkpointdirectory = Some(Map(&quot;pricePlanAuditBucket&quot; -&gt; &quot;PricePlanIngestionCheckpointDir&quot;,&quot;PricePlanS3backup&quot;-&gt;&quot;PricePlanS3backup&quot;)), kafkatopicmap = Some(Map.empty), schemamap = Some(Map.empty))<br><br>    val result = writeToS3(pricePlanDS)<br>    assert(result == false)<br>  }<br>4<br>  test(&quot;writeToS3 should not handle empty datset&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    var pricePlanDS = Seq.empty[PricePlanDocumentDBRecord].toDS()<br>    val result = writeToS3(pricePlanDS)<br>    assert(result == false)<br><br>  }<br><br>  test(&quot;writeToS3 should not handle exception&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    val pricePlanDS = pricePlanDetailsData.toDS()<br><br>    implicit val connectionConfigDetails: ConnectionConfigDetails = ConnectionConfigDetails(null)<br>    implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(jobname = &quot;PricePlanIngestion&quot;, jobidentifier = &quot;PricePlan&quot;, triggerinterval = Some(1000), kafkamaxrequestsize = Some(20971520), checkpointdirectory = Some(Map(&quot;pricePlanAuditBucket&quot; -&gt; &quot;PricePlanIngestionCheckpointDir&quot;,&quot;PricePlanS3backup&quot;-&gt;&quot;PricePlanS3backup&quot;)), kafkatopicmap = Some(Map.empty), schemamap = Some(Map.empty))<br><br>    val result = writeToS3(pricePlanDS)<br>    assert(result == true)<br><br>  }<br><br><br>  test(&quot;prepareDocumentDBReaderWithFilter should work correctly&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val expectedData = Seq(<br>      MyClass(&quot;value1&quot;),<br>      MyClass(&quot;value2&quot;),<br>      MyClass(&quot;value3&quot;)<br>    )<br><br>    // Call the function with the mock ReaderBuilder<br>    val actualDF = prepareDocumentDBReaderWithFilter(setupMockReaderBuilder(expectedData), &quot;testCollection&quot;, classOf[MyClass])<br><br>    // Check that the returned DataFrame is as expected<br>    actualDF.collect() shouldEqual expectedData.toDF().collect()<br>  }<br><br><br>  test(&quot;cleanupThirtyDayPeriodOne should exclude plans with end date not after 30 days&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br>//sc<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br>//    val testData = Seq(sampleMergedPricePlanMessage)<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>    assert(result.count() == 0)<br>  }<br><br>  test(&quot;cleanupThirtyDayPeriodOne should return type PricePlanDocumentDB&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    //val pricePlanDS = pricePlanDetailsData.toDS()<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>    assert(result.isInstanceOf[Dataset[PricePlanDocumentDBRecord]])<br>  }<br><br>  test(&quot;cleanupThirtyDayPeriodOne should include plans with end date after 30 days&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      newSamplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    //val pricePlanDS = pricePlanDetailsData.toDS()<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>     assert(result.count() == 0)<br>  }<br><br>  test(&quot;Testing functionality of convertToEntity&quot;) {<br><br>    val inputDataStr = scala.io.Source.fromFile(&quot;PricePlanIngestion/src/test/resources/input.json&quot;).mkString<br>    val ite: MergedPricePlan = convertToEntity(inputDataStr)(sparkSession1)<br>    assert(ite != null)<br>  }<br><br>  test(&quot;Testing error scenario of convertToEntity&quot;) {<br>    implicit val spark = sparkSession1<br>    val inputDataStr = scala.io.Source.fromFile(&quot;PricePlanIngestion/src/test/resources/inputWrong.json&quot;).mkString<br>    val ite: MergedPricePlan = convertToEntity(inputDataStr)<br>    assert(ite.pricePlan == PricePlan(null, null, null))<br>  }<br><br><br>  test(&quot;filterPropertyCode&quot;) {<br>    implicit val spark = sparkSession1<br>    val testData = Seq(sampleMergedPricePlanMessage)<br>    import spark.implicits._<br>    val testDS = testData.toDS()<br>    withObjectMocked[FilterPropertyCodeUtil.type]{<br>      when(<br>        FilterPropertyCodeUtil.filterPropertyCode(<br>          ArgumentMatchers.any(classOf[DataFrame]),<br>          ArgumentMatchers.any(classOf[Column])<br>        )<br>      ).thenReturn((testDS.toDF(), testDS.toDF()))<br>      val res = filterPropertyCode(testDS)<br>      assert(res.isInstanceOf[Dataset[MergedPricePlan]])<br>    }<br><br>  }<br><br>  test(&quot;mandatoryFieldValidation_should validate records&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br>    val sampleData = Seq(sampleMergedPricePlanMessage).toDS()<br>    val result = mandatoryFieldValidation(sampleData)<br>    result._1.head().pricePlan.propertyCode.code shouldEqual &quot;BOGAR&quot;<br>  }<br><br><br><br>  test(&quot;consumeDataFromEEH should call readStreamWithParams with correct parameters&quot;) {<br>    implicit val spark = sparkSession1<br>    val result = consumeDataFromEEH(InputType.Avro)<br>    assert(result != null)<br>  }<br><br><br>  test(&quot;consumeDataFromEEH should throw error and return empty dataframe&quot;) {<br>    implicit val spark = sparkSession1<br>    val result = consumeDataFromEEH(InputType.Json)<br>    assert(result === spark.emptyDataFrame)<br>  }<br><br>  test(&quot;generateKeyA should generate key correctly&quot;) {<br><br>    val propertycode = &quot;property1&quot;<br>    val pricePlanCode = &quot;pricePlan1&quot;<br>    val inventoryCode = &quot;code1&quot;<br><br>    val key = generateKeyA(propertycode, pricePlanCode, inventoryCode)<br><br>    assert(key == &quot;PRICEPLANDOC_property1_pricePlan1_code1&quot;)<br>  }<br><br><br><br>      test(&quot;write DS to Kafka functionality should frame message correctly&quot;) {<br>        implicit val spark = sparkSession1<br>        import spark.implicits._<br><br>        // Provide an instance of JobConfigurationDetails<br>        implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br>        val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br><br>        val genTimeDataset = Seq(<br>          (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>        ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>        val delRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>        val delRecDataframe = delRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br><br>        val initialNonDelRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>        // val delRecDataframe = delRecsDataset.withColumn()<br>        val NonDelRecsDataframe = initialNonDelRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br>          val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br>        // Flatten the combinedRecs DataFrame<br>        val flattenedCombinedRecs = inputDataset.select(<br>          $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>          $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>          $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;),<br>          $&quot;sourceString&quot;<br>        )<br><br>        // Join the flattened DataFrame with genTime<br>        val combinedWithGenTime = flattenedCombinedRecs.join(<br>          genTimeDataset,<br>          flattenedCombinedRecs(&quot;propertyCode&quot;) === genTimeDataset(&quot;propertyCode&quot;) &amp;&amp;<br>            flattenedCombinedRecs(&quot;pricePlanCode&quot;) === genTimeDataset(&quot;pricePlanCode&quot;),<br>          &quot;left&quot;<br>        ).withColumn(&quot;generationTime&quot;, coalesce(genTimeDataset(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>        // Call the function under test<br>        val result = writeDsToKafka(genTimeDataset, delRecDataframe, NonDelRecsDataframe, pricePlanRecsDataset)(spark, jobConfigDetails)<br><br>        // Assert that the result is a Dataset of String<br>        assert(result.isInstanceOf[Dataset[String]])<br><br><br>      }<br>    test(&quot;create docDB entity&quot;) {<br>      val outPutDB = createPriceDocument(sampleMergedPricePlanMessage, generateKeyA(sampleMergedPricePlanMessage.pricePlan.propertyCode.code, sampleMergedPricePlanMessage.pricePlan.pricePlanDetails.code, sampleMergedPricePlanMessage.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>      assert(outPutDB._id == &quot;PRICEPLANDOC_BOGAR_P1_ABC&quot;)<br>    }<br><br>    test(&quot;entityForLatestRecord 2 should transform PricePlan into PricePlanDocumentDBRecord&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br><br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br><br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br><br>      val outputDataset: Dataset[PricePlanDocumentDBRecord] = entityForLatestRecord(inputDataset)(spark, jobConfigDetails)<br><br>      assert(outputDataset.isInstanceOf[Dataset[PricePlanDocumentDBRecord]])<br>      //assert(outputDataset.head()._id != null)<br><br>    }<br><br>    test(&quot;compareRatesDatasets should compare two datasets correctly&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br>      val pricePlan1 = sampleMergedPricePlanMessage<br>      val pricePlanDoc1 = sampleDocPricePlan<br><br>      val merged = Seq(pricePlan1).toDS()<br>      val documentDb = Seq(pricePlanDoc1).toDS()<br><br>      val comparisonResult = comparePriceDatasets(merged, documentDb)(spark)<br><br>      assert(comparisonResult.isInstanceOf[Dataset[PriceComparisonResult]])<br><br>      val outputArray = comparisonResult.collect()<br><br>      assert(outputArray.length == 1)<br>      assert(outputArray.head.equals == true)<br>    }<br>  test(&quot;produceDataToEEH should produce data and log info&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    // Provide an instance of JobConfigurationDetails<br>    implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br>    val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br><br>    val genTimeDataset = Seq(<br>      (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>    ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>    val delRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>    val delRecDataframe = delRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br><br>    val initialNonDelRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>    // val delRecDataframe = delRecsDataset.withColumn()<br>    val NonDelRecsDataframe = initialNonDelRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br>    val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br>    // Flatten the combinedRecs DataFrame<br>    val flattenedCombinedRecs = inputDataset.select(<br>      $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>      $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>      $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;),<br>      $&quot;sourceString&quot;<br>    )<br><br>    // Join the flattened DataFrame with genTime<br>    val combinedWithGenTime = flattenedCombinedRecs.join(<br>      genTimeDataset,<br>      flattenedCombinedRecs(&quot;propertyCode&quot;) === genTimeDataset(&quot;propertyCode&quot;) &amp;&amp;<br>        flattenedCombinedRecs(&quot;pricePlanCode&quot;) === genTimeDataset(&quot;pricePlanCode&quot;),<br>      &quot;left&quot;<br>    ).withColumn(&quot;generationTime&quot;, coalesce(genTimeDataset(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>    // Call the function under test<br>    val result = writeDsToKafka(genTimeDataset, delRecDataframe, NonDelRecsDataframe, pricePlanRecsDataset)(spark, jobConfigDetails).select(from_json(col(&quot;PricePlanEventJson&quot;), avroSchema).alias(&quot;value&quot;))<br>        val res2 = result.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>          .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>          .withColumn(&quot;timestampType&quot;, lit(0))<br><br><br>        //when(producerHelper.produce()).thenReturn(Some(ProducerOutput(Some(true), None)))<br><br>        //val producer = new Producer(dataFrame, producerHelper, logger, exceptionHandler)<br>        val a = produceDataToEEH(res2)<br><br>        assert(a != null)<br>        //  verify(logger).info(any[String], any[CustomMeta])<br>  }<br><br><br>    test(&quot;process priceplan batch for latest rec&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage, sampleMergedPricePlanMessage2).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processPricePlanRecords(inputDataset)<br><br>      assert(res != null)<br><br>    }<br><br><br>    test(&quot;process non del flow for new rec&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with same data&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with new data older than 30 days&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleNewDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with new data less than 30 days&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleNewDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process del flow for new rec&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br>      val res = processDeletePricePlanDBRecords(inputDataset)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;Irrcoverable data should produce data and log info&quot;) {<br><br>      //val producerHelper = mock[ProducerHelper[EEHData]]<br><br><br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br><br>      // Create a MemoryStream of strings<br>      val stream = new MemoryStream[MergedPricePlan](1, spark.sqlContext)<br><br><br>      stream.addData(Seq(MergedPricePlan(PricePlan(null, null, null), &quot;abc&quot;)))<br><br>      // Create a DataFrame from the MemoryStream<br>      val invalidDF = stream.toDS()<br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>      //val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br><br>      val res = writeIrrErrorToKafka(invalidDF, jobConfigurationDetails.kafkatopicmap.get(IrrecoverableErrorTopic), S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(defaultBucket), jobName,<br>        jobConfigurationDetails.checkpointdirectory.get(IrrecoverableErrorCheckpointDir)))<br>      //when(producerHelper.produce()).thenReturn(Some(ProducerOutput(Some(true), None)))<br><br>      //val producer = new Producer(dataFrame, producerHelper, logger, exceptionHandler)<br>      assert(res != null)<br><br>      //  verify(logger).info(any[String], any[CustomMeta])<br>    }<br><br>    test(&quot;recoverable data should produce data and log info&quot;) {<br><br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>      val res = singleToRecoverable(inputDataset)(spark)<br>      assert(res != null)<br>    }<br><br>    test(&quot;object of source mmf topic&quot;) {<br>      implicit val spark = sparkSession1<br>      val res = com.marriott.mmf.PricePlan(<br>        PropertyChain(Some(&quot;chain&quot;), Some(&quot;MAR&quot;)), PropertyCode(Some(&quot;property&quot;), &quot;MCICO&quot;), Some(&quot;2024-05-03T07:57:15.627Z&quot;),<br>        SourceCRSEnum.ACRS,<br>        Some(LOSPricing(<br>          Some(LOSPricingActionEnum.Create), Some(&quot;basepricegrid&quot;), null, Some(Seq(<br>            LOSPriceGrid(null, Some(com.marriott.mmf.Range(null, null)))<br>          ))<br>        )),<br>        com.marriott.mmf.PricePlanDetails(<br>          Some(&quot;CAB1&quot;), Some(RatePlanActionEnum.Delete), null,<br>          Some(com.marriott.mmf.Promotion(<br>            null, null, null, Some(&quot;amountOff&quot;), null, null<br>          )<br>          ),<br>          Some(com.marriott.mmf.PriceGridDetails(<br>            null, Some(PriceGridTypeEnum.FlatRate), null,<br>            Some(TaxInclusionEnum.House), null,<br>            Some(CurrencyAmount(<br>              null, 12345, null, null<br>            )),<br>            Some(CurrencyAmount(<br>              null, 12345, null, null<br>            )),<br>            null, null, null, null,<br>            Some(Seq(<br>              AmountRule(null, null,<br>                Some(com.marriott.mmf.Timespan(Some(&quot;start&quot;), null, null)),<br>                Some(Seq(DowEnum.Friday)),<br>                Some(Seq(com.marriott.mmf.PricePoint(null, null, null, null, null,<br>                  Some(MarketType(null, Some(Description(<br>                    Some(Locale(<br>                      null, Some(&quot;USA&quot;)<br>                    )), Seq(&quot;abc&quot;), null<br>                  )), null)), null, null, null, null, null, null, null, Some(com.marriott.mmf.GuestsByType(Some(&quot;str&quot;), null))<br>                ))), null, null, null<br>              )<br>            )), null, null, null, null, null<br><br>          )<br>          ),<br>          Some(com.marriott.mmf.ProtectionLevel(<br>            Some(&quot;protectPricing&quot;), null<br>          )),<br>          Some(com.marriott.mmf.YieldManagement(<br>            null, null, null, null, Some(&quot;percent&quot;)<br>          )),<br>          Some(Seq(<br>            RatePeriod(<br>              null, null, Some(&quot;rounding&quot;), null, null, null, null, null, null, null, null, null, null, null<br>            )<br>          )),<br>          Some(com.marriott.mmf.StayRestriction(<br>            null, null, null, Some(&quot;invTypeCode&quot;), null, null, null, null, null, null, null, null, null, null, null, null,<br>            Some(RoomBucket(<br>              Some(&quot;roomBuck&quot;), null<br>            )), null, Some(Seq(<br>              ProtectionLock(<br>                Some(&quot;string&quot;), null<br>              )<br>            ))<br>          )),<br>          Some(&quot;refPriceGridCode&quot;),<br>          Some(com.marriott.mmf.TriggeringAction(<br>            null, Some(&quot;ratePlan&quot;), null, null, null<br>          )),<br>          Some(com.marriott.mmf.Pricing(<br>            Some(PricingFrequencyEnum.PerNight), null, null<br>          )),<br>          Some(Seq(<br>            TaxDetails(<br>              null, null, null, null, null, false,<br>              Some(DependencyAttribute(<br>                Some(&quot;attributeCode&quot;), null<br>              )), null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,<br>              null, null, null, null, null, null, Some(TaxScope(<br>                Some(PropertyChain(<br>                  Some(&quot;chain&quot;), null<br>                )), null<br>              )), Some(Seq(TaxItem(<br>                null, Some(&quot;CodeType&quot;), null<br>              ))), Some(Seq(TaxPeriod(<br>                null, null, Some(false), Some(Seq(TaxValue(<br>                  null, Some(&quot;percentVal&quot;), null, null, null, Some(Seq(PriceRange(<br>                    null, Some(CurrencyAmount(<br>                      null, 12345, null, null<br>                    ))<br>                  ))), null, null<br>                )))<br>              ))), null, Some(Seq(TaxOverrideScope(<br>                Some(&quot;code&quot;), null<br>              ))), null, null, null<br>            )<br>          )),<br>          Some(Seq(ClassificationEnum.Addon)),<br>          Some(&quot;currCode&quot;)<br>        )<br>      )<br>      assert(res.isInstanceOf[com.marriott.mmf.PricePlan])<br>    }<br>    test(&quot;amount rule = none &quot;) {<br>      implicit val spark = sparkSession1<br>      val res = com.marriott.mmf.PricePlan(chainCode = PropertyChain(), propertyCode = PropertyCode(code = &quot;code&quot;), pricePlanDetails = com.marriott.mmf.PricePlanDetails(), sourceCRSEnum = SourceCRSEnum.ACRS)<br>      val res1 = AmountRule()<br>      val res2 = CurrencyAmount(value = 12345)<br>      val res3 = DependencyAttribute()<br>      val res4 = Description(text = Seq(&quot;abc&quot;))<br>      val res5 = com.marriott.mmf.GuestsByType()<br>      val res6 = com.marriott.mmf.Locale()<br>      val res7 = LOSPriceGrid()<br>      val res8 = LOSPricing()<br>      val res9 = MarketType()<br>      val res10 = com.marriott.mmf.PriceGridDetails()<br>      val res11 = com.marriott.mmf.PricePoint()<br>      val res12 = com.marriott.mmf.PriceRange()<br>      val res13 = com.marriott.mmf.Pricing()<br>      val res14 = com.marriott.mmf.Promotion()<br>      val res15 = com.marriott.mmf.ProtectionLevel()<br>      val res16 = com.marriott.mmf.ProtectionLock()<br>      val res17 = com.marriott.mmf.Range()<br>      val res18 = com.marriott.mmf.RatePeriod()<br>      val res19 = com.marriott.mmf.RoomBucket()<br>      val res20 = com.marriott.mmf.StayRestriction()<br>      val res21 = com.marriott.mmf.TaxDetails(feeInd = true)<br>      val res22 = com.marriott.mmf.TaxItem()<br>      val res23 = com.marriott.mmf.TaxOverrideScope()<br>      val res24 = com.marriott.mmf.TaxPeriod()<br>      val res25 = com.marriott.mmf.TaxScope()<br>      val res26 = com.marriott.mmf.TaxValue()<br>      val res27 = com.marriott.mmf.Timespan()<br>      val res28 = com.marriott.mmf.TriggeringAction()<br>      val res29 = com.marriott.mmf.YieldManagement()<br>      assert(res.isInstanceOf[com.marriott.mmf.PricePlan])<br>      assert(res1.isInstanceOf[AmountRule])<br>      assert(res2.isInstanceOf[CurrencyAmount])<br>      assert(res3.isInstanceOf[DependencyAttribute])<br>      assert(res4.isInstanceOf[Description])<br>      assert(res5.isInstanceOf[com.marriott.mmf.GuestsByType])<br>      assert(res6.isInstanceOf[com.marriott.mmf.Locale])<br>      assert(res7.isInstanceOf[LOSPriceGrid])<br>      assert(res8.isInstanceOf[LOSPricing])<br>      assert(res9.isInstanceOf[MarketType])<br>      assert(res10.isInstanceOf[com.marriott.mmf.PriceGridDetails])<br>      assert(res11.isInstanceOf[com.marriott.mmf.PricePoint])<br>      assert(res12.isInstanceOf[com.marriott.mmf.PriceRange])<br>      assert(res13.isInstanceOf[com.marriott.mmf.Pricing])<br>      assert(res14.isInstanceOf[com.marriott.mmf.Promotion])<br>      assert(res15.isInstanceOf[com.marriott.mmf.ProtectionLevel])<br>      assert(res16.isInstanceOf[com.marriott.mmf.ProtectionLock])<br>      assert(res17.isInstanceOf[com.marriott.mmf.Range])<br>      assert(res18.isInstanceOf[com.marriott.mmf.RatePeriod])<br>      assert(res19.isInstanceOf[com.marriott.mmf.RoomBucket])<br>      assert(res20.isInstanceOf[com.marriott.mmf.StayRestriction])<br>      assert(res21.isInstanceOf[com.marriott.mmf.TaxDetails])<br>      assert(res22.isInstanceOf[com.marriott.mmf.TaxItem])<br>      assert(res23.isInstanceOf[com.marriott.mmf.TaxOverrideScope])<br>      assert(res24.isInstanceOf[com.marriott.mmf.TaxPeriod])<br>      assert(res25.isInstanceOf[com.marriott.mmf.TaxScope])<br>      assert(res26.isInstanceOf[com.marriott.mmf.TaxValue])<br>      assert(res27.isInstanceOf[com.marriott.mmf.Timespan])<br>      assert(res28.isInstanceOf[com.marriott.mmf.TriggeringAction])<br>      assert(res29.isInstanceOf[com.marriott.mmf.YieldManagement])<br><br>    }<br><br>  test(&quot;Split the data for Inventory Rate periods&quot;) {<br>    implicit val sparkSession = sparkSession1<br>    import sparkSession.implicits._<br>    implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br>    val sampleData = Seq(sampleMergedPricePlanChunkByInvMessage).toDS()<br>    val splitDOWRecs = splitDOW(sampleData)<br>    assert(!splitDOWRecs.filter(_.pricePlan.pricePlanDetails.ratePeriods.size &gt; 1).isEmpty)<br>  }<br><br>  test(&quot;Split the data for multiple Inventory &quot;) {<br>    implicit val sparkSession = sparkSession1<br>    import sparkSession.implicits._<br>    implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br>    val sampleData = Seq(updatedMergedPricePlanChunkByInvMessage).toDS()<br>    val splitByInv = chunkByInventory(sampleData)<br>    assert(!splitByInv.filter(_.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.size== 1).isEmpty)<br>  }<br><br>}<br>//  val genTime = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode1&quot;, &quot;pricePlanCode1&quot;, &quot;generationTime1&quot;)<br>//  )).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br>//<br>//  val delRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode1&quot;, &quot;pricePlanDetails1&quot;)<br>//  )).toDF(&quot;pricePlanDetails.propertyCode&quot;, &quot;pricePlanDetails.code&quot;)<br>//<br>//  val nonDelRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode2&quot;, &quot;pricePlanDetails2&quot;)<br>//  )).toDF(&quot;pricePlanDetails.propertyCode&quot;, &quot;pricePlanDetails.code&quot;)<br>//<br>//  val pricePlanRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;pricePlanDetails&quot;, &quot;generationTime&quot;)<br>//  )).toDF(&quot;pricePlan.propertyCode.code&quot;, &quot;pricePlan.pricePlanDetails.code&quot;, &quot;pricePlan.pricePlanDetails&quot;, &quot;pricePlan.generationTime&quot;)<br>//<br>//  implicit val jobConfigurationDetails: JobConfigurationDetails = mock[JobConfigurationDetails]<br>//  when(jobConfigurationDetails.kafkatopicmap).thenReturn(Some(Map(&quot;DestinationTopic&quot; -&gt; &quot;test-topic&quot;)))<br>//<br>//  //val kafkaProducer = new PricePlanIngestionHelper<br>//  val result: Dataset[String] = writeDsToKafka(genTime, delRecs, nonDelRecs, pricePlanRecs)<br>//<br>//  // Assertions<br>//  result.collect().foreach { jsonString =&gt;<br>//    jsonString should include(&quot;propertyCode&quot;)<br>//    jsonString should include(&quot;pricePlanId&quot;)<br>//    jsonString should include(&quot;isActive&quot;)<br>//  }}<br>//  test(&quot;write DS to kAfka functionality should frame message correctly&quot;) {<br>//    implicit val spark = sparkSession1<br>//    import spark.implicits._<br><br>// Provide an instance of JobConfigurationDetails<br>// implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>//    val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>//<br>//    // Dummy datasets for the other parameters<br>//    val genTimeDataset = Seq(<br>//      (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>//    ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br>//<br>//    val delRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().toDF()<br>//    val initialNonDelRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().toDF()<br>//    val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().flatMap { pricePlanElem =&gt;<br>//      pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.flatMap { ratePeriod =&gt;<br>//        ratePeriod.inventoryTypes.map { pricePoint =&gt;<br>//          (pricePlanElem.pricePlan.propertyCode.code,<br>//            pricePlanElem.pricePlan.pricePlanDetails.code,<br>//            ratePeriod,<br>//            pricePlanElem.pricePlan.generationTime,<br>//            pricePlanElem.sourceString)<br>//        }<br>//      }<br>//    }.toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;ratePeriods&quot;, &quot;generationTime&quot;, &quot;sourceString&quot;)<br>//<br>//    // Call the function under test<br>//    val result = writeDsToKafka(genTimeDataset, delRecsDataset, initialNonDelRecsDataset, pricePlanRecsDataset)(spark, jobConfigDetails)<br>//<br>//    // Assert that the result is a Dataset of String<br>//    assert(result.isInstanceOf[Dataset[String]])<br><br><br><br><br><br><br><br><br><br><br><br><br>TestHelper:<br><br>package com.marriott.mgpapp.spark.priceplan.test.helper<br><br>import com.marriott.elevate.spark.framework.config._<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.Default<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.{CurrValue, CurrencyAmount, GuestsByType, MergedPricePlan, PriceGridDetails, PriceGridDetailsDB, PricePlan, PricePlanDetails, PricePlanDetailsDB, PricePlanDocumentDBRecord, PricePoint, PricePointDB, Pricing, PropertyCode, RatePeriod, RatePeriodDB, StayRestriction, TieredRate, Timespan, YieldManagement, YieldManagementDB}<br><br>import scala.collection.Seq<br>trait TestHelper {<br><br><br>  implicit val PricePlanIngestion = &quot;PricePlanIngestion&quot;<br>  val pricePlan = &quot;pricePlan&quot;<br>  implicit val SchemaName = &quot;SchemaName&quot;<br><br>  implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(<br>    jobname = PricePlanIngestion,<br>    jobidentifier = pricePlan,<br>    triggerinterval = Some(1000),<br>    kafkamaxrequestsize = Some(20971520),<br>    checkpointdirectory = Some(Map(&quot;ReservationFiltrationCheckpointDir&quot; -&gt; &quot;ReservationFiltrationCheckpointDir&quot;,<br>      &quot;ReservationS3backup&quot; -&gt; &quot;ReservationS3backup&quot;,<br>      &quot;IrrecoverableErrorCheckpointDir&quot; -&gt; &quot;IrrecoverableErrorCheckpointDir&quot;)),<br>    kafkatopicmap = Some(Map(&quot;DestinationTopic&quot; -&gt; &quot;crs.reservation.event&quot;,<br>      &quot;RecoverableErrorTopic&quot; -&gt; &quot;commerce.reservation.price-plan.sync.revenue-management.one-yield.recoverable-test.dlt&quot;,<br>      &quot;IrrecoverableErrorTopic&quot; -&gt; &quot;commerce.reservation.price-plan.sync.revenue-management.one-yield.irrecoverable-test.dlt&quot;)),<br>    schemamap = Some(Map(&quot;PricePlanBucket&quot; -&gt; &quot;RMS-Rates&quot;,&quot;PricePlanCollection&quot;-&gt;&quot;PricePlan&quot;)), // Empty map for now as no example provided in the configuration<br>    doucmentdbmap = Some(Map(&quot;documentcollectionname&quot; -&gt; &quot;PricePlan&quot;))<br>  )<br>  val jobName: String = &quot;PricePlanIngestion&quot;<br>  val jobIdentifier: String = &quot;PricePlanIngestion&quot;<br>  val triggerInterval: Int = 2000<br>  implicit val connectionConfigDetails: ConnectionConfigDetails =<br>    ConnectionConfigDetails(Some(DbDetails(<br>      jdbcurl = &quot;jdbc:mysql://localhost:3306/mydb&quot;,<br>      drivername = &quot;com.mysql.jdbc.Driver&quot;,<br>      username = &quot;my_username&quot;,<br>      password = &quot;my_password&quot;,<br>      poolsize = &quot;10&quot;,<br>      idletimeout = 10000, // Milliseconds<br>      jdbcurlro = &quot;jdbc:mysql://localhost:3306/mydb_read_only&quot;,<br>      drivernamero = &quot;com.mysql.jdbc.Driver&quot;,<br>      usernamero = &quot;ro_username&quot;,<br>      passwordro = &quot;ro_password&quot;,<br>      poolsizero = &quot;5&quot;,<br>      idletimeoutro = 1500, // Milliseconds<br>      jdbcurlwo = &quot;jdbc:mysql://localhost:3306/mydb_write_only&quot;,<br>      drivernamewo = &quot;com.mysql.jdbc.Driver&quot;,<br>      usernamewo = &quot;wo_username&quot;,<br>      passwordwo = &quot;wo_password&quot;,<br>      poolsizewo = &quot;8&quot;,<br>      idletimeoutwo = 12000L // Milliseconds<br>    )),<br>      Some(DocumentDbDetails(<br>        host = Seq(&quot;rms-oy-dev-documentdb.cluster-cbgemarietsq.us-east-1.docdb.amazonaws.com:27017&quot;).toList,<br>        username = &quot;user&quot;,<br>        password = &quot;password&quot;,<br>        db = &quot;RMS-RES-JOURNAL&quot;,<br>        collection = &quot;default&quot;,<br>        connectiontimeout = &quot;600000&quot;<br>      )),<br>      Some(MessageBrokerDetails(<br>        bootstrapserver = &quot;localhost:9092&quot;,<br>        sourcetopic = &quot;source_topic&quot;,<br>        destinationtopic = &quot;destination_topic&quot;,<br>        recoverableerrortopic = &quot;recoverable_error_topic&quot;,<br>        irrecoverableerrortopic = &quot;irrecoverable_error_topic&quot;,<br>        kafkakeyserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;,<br>        kafkavalueserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;,<br>        maxOffsetTriggerInterval = Some(1000), // Some specific value (milliseconds)<br>        kafkasecurityprotocol = None, // Using the default value (None)<br>        kafkassltruststorelocation = None, // Using the default value (None)<br>        kafkasslkeystorelocation = None, // Using the default value (None)<br>        kafkassltruststorepassword = None, // Using the default value (None)<br>        kafkasslkeystorepassword = None, // Using the default value (None)<br>        kafkaidentificationalgorithm = None, // Using the default value (None)<br>        kafkatruststoretype = None, // Using the default value (None)<br>        pollTimeoutMs = Some(5000) // Some specific value (milliseconds)<br>      )),<br>      Some(Map(&quot;default&quot; -&gt; &quot;Checkpoint\\Bucket&quot;)), Some(Map()), Some(Map()))<br><br>  val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)<br>  implicit val checkpointDirMap: Map[String, String] = jobConfigurationDetails.checkpointdirectory.get<br>  implicit val checkpointPath: Option[String] = Some(&quot;Checkpoint\\Bucket&quot;)<br><br><br><br><br>  val sampleMergedPricePlanMessage2 = MergedPricePlan(PricePlan(<br>    propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>    generationTime = &quot;2021-01-02&quot;,<br>    PricePlanDetails(<br>      &quot;P1&quot;,<br>      Some(&quot;CREATE&quot;),<br>      Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetails(Some(&quot;Standard&quot;), Some(&quot;Regular&quot;), Some(true), Some(&quot;Inclusion 1&quot;), Some(&quot;Category 1&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;priceendswith&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(1), Seq(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagement(Some(false), Some(&quot;type&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;percentage&quot;))),<br>      Seq(RatePeriod(Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>        Some(Timespan(Some(&quot;start&quot;), Some(&quot;2024-05-30&quot;), Some(&quot;duration&quot;))),<br>        Some(&quot;down&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;10&quot;),<br>        Some(&quot;10&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;level&quot;),<br>        Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>        Some(1),<br>        Some(&quot;guestTypeEnum&quot;), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>      &quot;USD&quot;<br>    )<br>  ), &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;)<br><br><br>  val sampleDocPricePlan = PricePlanDocumentDBRecord(&quot;PRICEPLANNDOC_BOGAR_P1_ABC&quot;,<br>    PricePlanDetailsDB(&quot;P1&quot;,&quot;BOGAR&quot;,&quot;ABC&quot;,Some(&quot;CREATE&quot;),Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetailsDB(Some(&quot;Standard&quot;),Some(&quot;Regular&quot;),Some(true),Some(CurrValue(12.34)),Some(&quot;Inclusion 1&quot;),Some(&quot;usd&quot;),Some(&quot;priceendswith&quot;),Some(&quot;Category 1&quot;),Some(&quot;classCode&quot;),Some(&quot;pCode&quot;),Some(CurrValue(12.34)),Some(1),List(GuestsByType(Some(&quot;guest&quot;),Some(1)))) ),<br>      Some(YieldManagementDB(Some(false),Some(&quot;type&quot;),Some(CurrValue(12.34)),Some(&quot;percentage&quot;),Some(CurrValue(12.34))) ),<br>      Seq(RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;),Some(&quot;down&quot;),Some(Seq(TieredRate(null,null,null,null,null))),12.34,12.34,null,null,Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(&quot;10&quot;),Some(&quot;10&quot;),Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(&quot;level&quot;),Some(1),Some(&quot;guestTypeEnum&quot;),<br>        Seq(PricePointDB(&quot;ABC&quot;,Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(false),Some(1),Some(&quot;guestTypeEnum&quot;),Some(GuestsByType(Some(&quot;guest&quot;),Some(1))))),Some(Timespan(Some(&quot;start&quot;),Some(&quot;end&quot;),Some(&quot;duration&quot;))), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq( com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;),Some(&quot;max&quot;))),Seq(&quot;string&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;)) ),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;),Some(&quot;methods&quot;),Some(&quot;book&quot;)) ),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),&quot;USD&quot;),null)<br><br>  val sampleNewDocPricePlan = PricePlanDocumentDBRecord(&quot;PRICEPLANNDOC_BOGAR_P1_ABC&quot;,<br>    PricePlanDetailsDB(&quot;P1&quot;, &quot;BOGAR&quot;, &quot;ABC&quot;, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetailsDB(Some(&quot;New&quot;), Some(&quot;Regular&quot;), Some(true), Some(CurrValue(12.34)), Some(&quot;Inclusion 1&quot;), Some(&quot;usd&quot;), Some(&quot;priceendswith&quot;), Some(&quot;Category 1&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(CurrValue(12.34)), Some(1), List(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagementDB(Some(false), Some(&quot;type&quot;), Some(CurrValue(12.34)), Some(&quot;percentage&quot;), Some(CurrValue(12.34)))),<br>      Seq(RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;), Some(&quot;down&quot;), Some(Seq(TieredRate(null, null, null, null,null))), 12.34, 12.34, null, null, Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(&quot;level&quot;), Some(1), Some(&quot;guestTypeEnum&quot;),<br>        Seq(PricePointDB(&quot;ABC&quot;, Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(false), Some(1), Some(&quot;guestTypeEnum&quot;), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))), Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;), &quot;USD&quot;), null)<br><br>  val sampleDelMessage = PricePlan(<br>    propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>    generationTime = &quot;2021-01-01&quot;,<br>    PricePlanDetails(<br>      &quot;P1&quot;,<br>      Some(&quot;Delete&quot;),<br>      Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetails(Some(&quot;Standard&quot;), Some(&quot;Regular&quot;), Some(true), Some(&quot;Inclusion 1&quot;), Some(&quot;Category 1&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;priceendswith&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(1), Seq(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagement(Some(false), Some(&quot;type&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;percentage&quot;))),<br>      Seq(RatePeriod(Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>        Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))),<br>        Some(&quot;down&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;10&quot;),<br>        Some(&quot;10&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;level&quot;),<br>        Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>        Some(1),<br>        Some(&quot;guestTypeEnum&quot;), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>      &quot;USD&quot;<br>    )<br>  )<br><br><br><br><br><br><br><br>  val sampleMergedPricePlanMessage = MergedPricePlan(<br>     PricePlan(<br>      propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>      generationTime = &quot;2021-01-01&quot;,<br>      pricePlanDetails = PricePlanDetails(<br>        code = &quot;P1&quot;,<br>        actionEnum = Some(&quot;CREATE&quot;),<br>        statusEnum = Some(&quot;ACTIVE&quot;),<br>        priceGridDetails = Some(PriceGridDetails(<br>          name = Some(&quot;Standard&quot;),<br>          typeEnum = Some(&quot;Regular&quot;),<br>          applicableToAllInventoryTypesInd = Some(true),<br>          taxInclusionEnum = Some(&quot;Inclusion 1&quot;),<br>          productCategoryCode = Some(&quot;Category 1&quot;),<br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          priceEndsWith = Some(&quot;priceendswith&quot;),<br>          productClassCode = Some(&quot;classCode&quot;),<br>          productCode = Some(&quot;pCode&quot;),<br>          maxFreeNonAdults = Some(1),<br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br>        )),<br>         Some(YieldManagement(<br>          yieldChangedInd = Some(false),<br>          typeEnum = Some(&quot;type&quot;),<br>          flatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percent = Some(&quot;percentage&quot;)<br>        )),<br>        Seq(RatePeriod(<br>          activeDowEnum = Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>          timespan = Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))),<br>          rounding = Some(&quot;down&quot;),<br>          amountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          amountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percentAdd = Some(&quot;10&quot;),<br>          percentOff = Some(&quot;10&quot;),<br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localAmountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          level = Some(&quot;level&quot;),<br>          inventoryTypes = Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>          occupancy = Some(1),<br>          guestTypeEnum = Some(&quot;guestTypeEnum&quot;),<br>          Some(&quot;USD&quot;)<br>        )),<br>         Some(StayRestriction(Seq( com.marriott.mgpapp.spark.priceplan.<br>          ingestion.entity.Range(Some(&quot;min&quot;),Some(&quot;max&quot;))),Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;) )),<br>        referencePriceGridCode = Some(&quot;refPriceGridCode&quot;),<br>        pricing = Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>        classificationsEnum = Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>        currencyCode = &quot;USD&quot;<br>      )<br>    ),<br>    &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;<br>  )<br><br>  val sampleMergedPricePlanChunkByInvMessage = MergedPricePlan(<br>    PricePlan(<br>      propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>      generationTime = &quot;2021-01-01&quot;,<br>      pricePlanDetails = PricePlanDetails(<br>        code = &quot;P1&quot;,<br>        actionEnum = Some(&quot;CREATE&quot;),<br>        statusEnum = Some(&quot;ACTIVE&quot;),<br>        priceGridDetails = Some(PriceGridDetails(<br>          name = Some(&quot;Standard&quot;),<br>          typeEnum = Some(&quot;Regular&quot;),<br>          applicableToAllInventoryTypesInd = Some(true),<br>          taxInclusionEnum = Some(&quot;Inclusion 1&quot;),<br>          productCategoryCode = Some(&quot;Category 1&quot;),<br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          priceEndsWith = Some(&quot;priceendswith&quot;),<br>          productClassCode = Some(&quot;classCode&quot;),<br>          productCode = Some(&quot;pCode&quot;),<br>          maxFreeNonAdults = Some(1),<br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br>        )),<br>        Some(YieldManagement(<br>          yieldChangedInd = Some(false),<br>          typeEnum = Some(&quot;type&quot;),<br>          flatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percent = Some(&quot;percentage&quot;)<br>        )),<br>        Seq(RatePeriod(<br>          activeDowEnum = Seq(&quot;Monday&quot;, &quot;Tuesday&quot;),<br>          timespan = Some(Timespan(Some(&quot;2025-07-20&quot;), Some(&quot;2025-08-10&quot;), Some(&quot;10&quot;))),<br>          rounding = Some(&quot;down&quot;),<br>          amountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          amountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percentAdd = Some(&quot;10&quot;),<br>          percentOff = Some(&quot;10&quot;),<br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localAmountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          level = Some(&quot;level&quot;),<br>          inventoryTypes = Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>          occupancy = Some(1),<br>          guestTypeEnum = Some(&quot;guestTypeEnum&quot;),<br>          Some(&quot;USD&quot;)<br>        )),<br>        Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.<br>          ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>        referencePriceGridCode = Some(&quot;refPriceGridCode&quot;),<br>        pricing = Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>        classificationsEnum = Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>        currencyCode = &quot;USD&quot;<br>      )<br>    ),<br>    &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;<br>  )<br><br>  val updatedMergedPricePlanChunkByInvMessage = MergedPricePlan(<br><br>    PricePlan(<br><br>      propertyCode = PropertyCode(&quot;BOGAR&quot;), // Updated from &quot;BOGAR&quot;<br><br>      generationTime = &quot;2024-04-30T13:10:43.080Z&quot;, // Updated from &quot;2021-01-01&quot;<br><br>      pricePlanDetails = PricePlanDetails(<br><br>        code = &quot;DPP2&quot;, // Updated from &quot;P1&quot;<br><br>        actionEnum = Some(&quot;Create&quot;), // Updated from &quot;CREATE&quot;<br><br>        statusEnum = Some(&quot;Active&quot;), // Updated from &quot;ACTIVE&quot;<br><br>        priceGridDetails = Some(PriceGridDetails(<br><br>          name = Some(&quot;New price grid&quot;), // Updated from &quot;Standard&quot;<br><br>          typeEnum = Some(&quot;FlatRate&quot;), // Updated from &quot;Regular&quot;<br><br>          applicableToAllInventoryTypesInd = Some(true), // No change<br><br>          taxInclusionEnum = Some(&quot;IncludedInRate&quot;), // Updated from &quot;Inclusion 1&quot;<br><br>          productCategoryCode = Some(&quot;string&quot;), // Updated from &quot;Category 1&quot;<br><br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;USD&quot;), 30, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          priceEndsWith = Some(&quot;30&quot;), // Updated from &quot;priceendswith&quot;<br><br>          productClassCode = Some(&quot;SPA&quot;), // Updated from &quot;classCode&quot;<br><br>          productCode = Some(&quot;MASSAGE&quot;), // Updated from &quot;pCode&quot;<br><br>          maxFreeNonAdults = Some(0), // Updated from 1<br><br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;Teen&quot;), Some(2))) // Updated from Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br><br>        )),<br><br>        Some(YieldManagement(<br><br>          yieldChangedInd = Some(true), // Updated from false<br><br>          typeEnum = Some(&quot;AdjustmentValue&quot;), // Updated from &quot;type&quot;<br><br>          flatAmount = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          percent = Some(&quot;0.0&quot;) // Updated from &quot;percentage&quot;<br><br>        )),<br><br>        Seq(RatePeriod(<br><br>          activeDowEnum = Seq(&quot;Monday&quot;, &quot;Wednesday&quot;, &quot;Friday&quot;, &quot;Saturday&quot;, &quot;Sunday&quot;), // Updated from Seq(&quot;Monday&quot;, &quot;Tuesday&quot;)<br><br>          timespan = Some(Timespan(Some(&quot;2024-02-19&quot;), Some(&quot;2024-02-19&quot;), None)), // Updated from Some(&quot;2025-07-20&quot;), Some(&quot;2025-08-10&quot;), Some(&quot;10&quot;)<br><br>          rounding = Some(&quot;down&quot;), // No change<br><br>          amountAdd = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          amountOff = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          percentAdd = Some(&quot;0.0&quot;), // Updated from &quot;10&quot;<br><br>          percentOff = Some(&quot;0.0&quot;), // Updated from &quot;10&quot;<br><br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localAmountOff = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          level = Some(&quot;string&quot;), // Updated from &quot;level&quot;<br><br>          inventoryTypes = Seq(<br>            PricePoint(<br>              &quot;DIT2&quot;,<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(true), // Updated from false<br>              Some(GuestsByType(Some(&quot;Adult&quot;), Some(2))) // Updated from GuestsByType(Some(&quot;guest&quot;), Some(1))<br>            )<br>          ),<br><br>          occupancy = Some(2), // Updated from 1<br><br>          guestTypeEnum = Some(&quot;Adult&quot;), // Updated from &quot;guestTypeEnum&quot;<br>          Some(&quot;USD&quot;)<br>        )),<br><br>        Some(StayRestriction(<br>          Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;0&quot;), Some(&quot;0&quot;))), // Updated from Seq(Some(&quot;min&quot;), Some(&quot;max&quot;))<br>          Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;),<br>          Seq(&quot;String&quot;),<br>          Seq(&quot;String&quot;)<br>        )),<br><br>        referencePriceGridCode = Some(&quot;J4U&quot;), // Updated from &quot;refPriceGridCode&quot;<br><br>        pricing = Some(Pricing(<br>          Some(&quot;PerNight&quot;), // Updated from Some(&quot;freq&quot;)<br>          Some(&quot;PerPerson&quot;), // Updated from Some(&quot;methods&quot;)<br>          Some(&quot;DayOfCheckIn&quot;) // Updated from Some(&quot;book&quot;)<br>        )),<br><br>        classificationsEnum = Seq(&quot;Addon&quot;), // Updated from Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;)<br><br>        currencyCode = &quot;USD&quot; // No change<br><br>      )<br>    ), &quot;&quot;)<br><br><br>}</pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=38eb6c0aa0a4" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[FASTAPI-Backup 4]]></title>
            <link>https://medium.com/@ankitjuneja/fastapi-7e13425e2c7b?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/7e13425e2c7b</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Mon, 21 Apr 2025 18:45:08 GMT</pubDate>
            <atom:updated>2025-08-22T04:45:52.862Z</atom:updated>
            <content:encoded><![CDATA[<blockquote>Webframework for building API</blockquote><blockquote>Runing it through UVIcorn server</blockquote><p>Pydantic library which is data validation</p><p>pydantic covers the limitation of python not defining datatypes</p><p>pydantic will give the structure of input and the output we will be sending</p><p>for that we will be importing BaseModel below from pydantic library</p><p>Let us make a basic Webappilcation:</p><p>Go to vscode and go to Terminal</p><p>&gt; Python3 -m venv venv</p><p>&gt; source venv/bin/activate</p><p>now venv is activated</p><p>(venv)&gt;pip install fastAPI uvicorn</p><p>or also we can pip freeze and lock it in the requirement.txt</p><p>pip freeze &gt; requirements.txt</p><p>this will be stored in requiremnt.txt:</p><p>1.annotated-types==0.7.0</p><p>2.anyio==4.9.0</p><p>3.click==8.1.8</p><p>4.fastapi==0.115.12</p><p>5.h11=0.14.0</p><p>6.idna==3.10</p><p>7.pydantic==2.11.2</p><p>8.pydantic_core==2.33.1</p><p>9.sniffio==1.3.1</p><p>10.starlette==0.46.1</p><p>11.typing-inspection =0.4.0</p><p>12.typing_extensions==4.13.1</p><p>13.uvicorn==0.34.0</p><p>now we will write the code:</p><p>let create a file main.py</p><p>from fastapi import FastApi</p><p>from pydantic impor BaseModel</p><p>from typing import List</p><p>#pydantic will have 2 model :input from we b request and output from webrequest</p><p>app = FastApi()</p><p>#here you are using pydantic libray</p><p>class Tea(BaseModel):</p><p>id: str</p><p>name:str</p><p>origin:str</p><p>#basic array</p><p>teas = []</p><p>#but since you have imported the typing you could use the structure which will help:</p><p>teas :List[Tea] = []</p><p>#FastApi generally works on the principle of Decorators.</p><p>#example if some visits to our route for example / =&gt; the output should be like “Welcome to tea house”</p><p>we can have many routes</p><p>/</p><p>/teas</p><p>/teas/type</p><p>so for this we would be needing decorator</p><p>@app.get(“/”)</p><p>#now &lt;@app.get(“/”)&gt; is a decorator.just after this decorator we will be defining the method</p><p>def read_root():</p><p>return {“message: Welcome to home”}</p><p>Now similarly we will make anothe routes:</p><p>@app.get(“/teas”):</p><p>def get_teas():</p><p>return teas</p><p>#Now we will add data using Post.</p><p>#also since am using pydantic i can define the input type which i have defined above</p><p>@app.post(“/teas”)</p><p>def add_tea(tea:Tea):</p><p>teas.append(tea)</p><p>return tea</p><p>#put method:</p><p>@app.put (“/teas/{tea_id}”)</p><p>def update_tea(tea_id:int, updated_tea :Tea):</p><p>for index,tea in enumerate(teas):</p><p>if tea.id == tea_id:</p><p>teas[index] = update_tea</p><p>return update_tea</p><p>return {“error” :Tea not found”}</p><p>#for delete</p><p>@app.delete(“/delete/{tea_id}”)</p><p>def delete_tea(tea_id:int):</p><p>for index,tea in enumerate(teas):</p><p>if tea.id == tea_id:</p><p>deleted = teas.pop(index)</p><p>return deleted</p><p>return {“error”: “Tea not found”}</p><p>Now everything we run via app, how to run this, it is in the documentation</p><p>Currently we will run through uvicorn</p><p>go to terminal:</p><p>&gt; uvicorn main:app — -reload</p><p>#it will give us the host <a href="http://127.0.0.1.000">http://127.0.0.1.000</a></p><p>the screen is showing “ Welcome to home”</p><p>for put , post delete you need to use Postman</p><p>you can also use <a href="http://127.0.0.1.000">http://127.0.0.1.000</a>/docs</p><p>it will give you all the put post and delete method: you can add all the details here.</p><p>you can also add jwt token and all authentication:</p><p>=================</p><p>Rest ful api meaning : its the representation of data</p><h3>Backup -4</h3><p>Build.sbt</p><pre>/**<br>  * Use only root build.sbt for adding sub-projects and use Dependencies.scala to add specific dependency to the sub-project<br>  * Add sub-projects only which has to be enabled for CI / CD<br>  * Comment out projects which is in development phase<br>  */</pre><pre>lazy val OTBSnapshot =<br>  (project in file(&quot;OTBSnapshot&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;OTBSnapshot&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.OTBSnapshot)<br>//  .settings(dependencyOverrides ++= Dependencies.OTBSnapshotJobOverrides)</pre><pre>lazy val GroupRateProgramETL =<br>  (project in file(&quot;GroupRateProgramETL&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;GroupRateProgramETL&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.GroupRateProgramETL)<br>    .settings(CleanUpSlf4j.settings: _*)<br>    .configs(IntegrationTest.configs: _*)<br>    .settings(IntegrationTest.settings: _*)</pre><pre>lazy val GroupBlockedRoomsETL =<br>  (project in file(&quot;GroupBlockedRoomsETL&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;GroupBlockedRoomsETL&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.GroupBlockedRoomsETL)</pre><pre>lazy val TransientBookedAndAdditionalDemandAtDaysLeftDimentions =<br>  (project in file(&quot;TransientBookedAndAdditionalDemandAtDaysLeftDimentions&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;TransientBookedAndAdditionalDemandAtDaysLeftDimentions&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.TransientBookedAndAdditionalDemandAtDaysLeftDimentions)</pre><pre>lazy val MiniConversionPublish =<br>  (project in file(&quot;MiniConversionPublish&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;MiniConversionPublish&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.MiniConversionPublish)</pre><pre>lazy val GroupBookedAtDaysLeftDimension =<br>  (project in file(&quot;GroupBookedAtDaysLeftDimension&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;GroupBookedAtDaysLeftDimension&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.GroupBookedAtDaysLeftDimension)</pre><pre>lazy val GroupBlockedAndGroupForecastAtDaysLeftDimension =<br>  (project in file(&quot;GroupBlockedAndGroupForecastAtDaysLeftDimension&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;GroupBlockedAndGroupForecastAtDaysLeftDimension&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.GroupBlockedAndGroupForecastAtDaysLeftDimension)</pre><pre>lazy val RefreshImplement =<br>  (project in file(&quot;RefreshImplement&quot;))<br>    .settings(Common.settings: _*)<br>    .settings(assemblySettings)<br>    .settings(name := &quot;RefreshImplement&quot;)<br>    .settings(libraryDependencies ++= Dependencies.commonDependencies)<br>    .settings(libraryDependencies ++= Dependencies.RefreshImplement)<br></pre><pre>lazy val assemblySettings = assemblyMergeStrategy in assembly := {<br>  case m if m.toLowerCase.endsWith(&quot;manifest.mf&quot;) =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.matches(&quot;meta-inf.*\\.sf$&quot;) =&gt; MergeStrategy.discard<br>  case &quot;log4j.properties&quot; =&gt; MergeStrategy.discard<br>  case m if m.toLowerCase.startsWith(&quot;meta-inf/services/&quot;) =&gt;<br>    MergeStrategy.filterDistinctLines<br>  case &quot;reference.conf&quot; =&gt; MergeStrategy.concat<br>  case _ =&gt; MergeStrategy.first<br>}</pre><p>utility.py</p><pre>#!/usr/bin/python3<br>import decimal<br>import logging<br>import socket<br>import sys<br>import os<br>import json<br>from urllib.parse import urlparse<br>from builtins import KeyError, AttributeError<br>from datetime import date, datetime, timedelta<br>import configparser<br>from configparser import NoOptionError, NoSectionError<br>import requests<br>import boto3<br>import urllib3<br>from botocore.exceptions import ClientError<br>import psycopg2<br>from psycopg2.extras import RealDictCursor<br>from psycopg2 import OperationalError, IntegrityError, ProgrammingError, DatabaseError<br>from requests import adapters<br>from urllib3 import connection</pre><pre># pylint: disable=E0202</pre><pre># S3 CONSTANTS<br>S3_SECTION = &#39;s3&#39;<br>ACCESS_ID = &#39;ACCESS_ID&#39;<br>ACCESS_KEY = &#39;ACCESS_KEY&#39;</pre><pre># S3 CONFIG CONSTANTS<br>ACCESSID = &#39;accessid&#39;<br>ACCESSKEY = &#39;accesskey&#39;</pre><pre># DB CONSTANTS<br>DB_SECTION = &#39;database&#39;<br>SGDS_DB_SECTION = &#39;sgds_database&#39;<br>#####DB_DATABASE = &#39;DATABASE&#39;<br>#####DB_HOST = &#39;HOST&#39;<br>#####DB_PORT = &#39;PORT&#39;<br>#####DB_PASSWORD = &#39;PASSWORD&#39;<br>#####DB_USER = &#39;USER&#39;<br>#####PG_OPTION = &#39;PG_OPTION&#39;<br>READ_DB_DATABASE = &#39;READ_DATABASE&#39;<br>READ_DB_HOST = &#39;READ_HOST&#39;<br>READ_DB_PORT = &#39;READ_PORT&#39;<br>READ_DB_PASSWORD = &#39;READ_PASSWORD&#39;<br>READ_DB_USER = &#39;READ_USER&#39;<br>READ_PG_OPTION = &#39;READ_PG_OPTION&#39;<br>  ##Added below for EDB RW Separation<br>WRITE_DB_DATABASE = &#39;WRITE_DATABASE&#39;<br>WRITE_DB_HOST = &#39;WRITE_HOST&#39;<br>WRITE_DB_PORT = &#39;WRITE_PORT&#39;<br>WRITE_DB_PASSWORD = &#39;WRITE_PASSWORD&#39;<br>WRITE_DB_USER = &#39;WRITE_USER&#39;<br>WRITE_PG_OPTION = &#39;READ_PG_OPTION&#39;</pre><pre># DB CONFIG CONSTANTS</pre><pre>READ_DATABASE = &#39;read_database&#39;<br>READ_HOST = &#39;read_host&#39;<br>READ_PORT = &#39;read_port&#39;<br>READ_PASSWORD = &#39;read_password&#39;<br>READ_USER = &#39;read_user&#39;</pre><pre>WRITE_DATABASE = &#39;write_database&#39;<br>WRITE_HOST = &#39;write_host&#39;<br>WRITE_PORT = &#39;write_port&#39;<br>WRITE_PASSWORD = &#39;write_password&#39;<br>WRITE_USER = &#39;write_user&#39;</pre><pre># ENV CONFIG CONSTANTS<br>ENV_SECTION = &#39;env&#39;<br>DB_SCHEMA = &#39;SCHEMA&#39;<br>BUCKET_NAME = &#39;BUCKET_NAME&#39;<br>FILE_CATALOG = &#39;FILE_CATALOG&#39;</pre><pre># S3 ENVIRONMENT VARIABLES<br>AWS_ACCESS_KEY_ID = &#39;AWS_ACCESS_KEY_ID&#39;<br>AWS_SECRET_ACCESS_KEY = &#39;AWS_SECRET_ACCESS_KEY&#39;</pre><pre># DATABASE ENVIRONMENT VARIABLES<br>PGDATABASE = &#39;PGDATABASE&#39;<br>PGHOST = &#39;PGHOST&#39;<br>PGPORT = &#39;PGPORT&#39;<br>PGUSER = &#39;PGUSER&#39;<br>PGPASSWORD = &#39;PGPASSWORD&#39;<br>PGOPTIONS = &#39;PGOPTIONS&#39;</pre><pre># ENVIRONMENT VARIABLE FOR BUCKET AND SCHEMA<br>SCHEMA = &#39;SCHEMA&#39;<br>BUCKETNAME = &#39;BUCKETNAME&#39;<br>FILECATALOG = &#39;FILE_CATALOG&#39;<br>REQUESTS_CA_BUNDLE = &#39;REQUESTS_CA_BUNDLE&#39;<br># DELIMITER CONSTANTS<br>AND_DELIM = &#39;&amp;&#39;<br>COMMA_DELIM = &#39;,&#39;<br>DOT_DELIM = &#39;.&#39;<br>EQUALSTO = &#39;=&#39;<br>FOLDER_DELIM = &#39;/&#39;<br>HOST_DELIM = &#39;?&#39;<br>QUERY_DELIM = &#39;&amp;&#39;<br>QUEST_DELIM = &#39;?&#39;</pre><pre># JOB CONFIG INPUT DS LIST CONSTATNS<br>DS_DATASETS = &#39;datasets&#39;<br>DS_FEATUREID = &#39;featureid&#39;<br>DS_DATASETID = &#39;datasetid&#39;<br>DS_CONDITION = &#39;condition&#39;<br>DS_BYPROPERTY = &#39;byproperty&#39;<br>DS_ISMANDATE = &#39;ismandate&#39;<br># JOBCONFIG FILE CONSTANTS<br>INPUT_DS_LIST = &#39;input_ds_list&#39;</pre><pre># JSON FILE CONSTANTS INPUT DS<br>INPUTDATAPATH = &#39;inputdatapath&#39;<br>DATASETID = &#39;datasetid&#39;<br>DATASETPATH = &#39;datasetpath&#39;<br>SDSK = &#39;sdsk&#39;<br>RUNID = &#39;runid&#39;<br>ASOFDATE = &#39;asofdate&#39;<br>RUNDATE = &#39;rundate&#39;<br>DATAFROMDATE = &#39;datafromdate&#39;<br>DATATODATE = &#39;datatodate&#39;<br>DATADATEFROM = &#39;datadatefrom&#39;<br>DATADATETO = &#39;datadateto&#39;<br>APPLICABLEDATEFROM = &#39;applicabledatefrom&#39;<br>APPLICABLEDATETO = &#39;applicabledateto&#39;<br>WARNINGDATE = &#39;warningdate&#39;<br>VALIDFROM = &#39;validfrom&#39;<br>VALIDTO = &#39;validto&#39;</pre><pre># RMS_JSON_CONSTANT<br>SD_SK = &#39;sd_sk&#39;<br>SD_RUN_ID = &#39;sd_run_id&#39;<br>SD_DATASET_PATH = &#39;sd_dataset_path&#39;<br>SD_RUN_DATE = &#39;sd_run_date&#39;<br>SD_DATA_DATE_FROM = &#39;sd_data_date_from&#39;<br>SD_DATA_DATE_TO = &#39;sd_data_date_to&#39;<br>SD_VALID_FROM = &#39;sd_valid_from&#39;<br>SD_VALID_TO = &#39;sd_valid_to&#39;<br>SD_WARNING_DATE = &#39;sd_warning_date&#39;<br>SD_ASOF_DATE = &#39;sd_asof_date&#39;</pre><pre># JSON FILE CONSTANTS REFERENCE FILE<br>REFERENCEDATAPATH = &#39;referencedatapath&#39;<br>REFERENCEFILEID = &#39;referencefileid&#39;<br>REFERENCEPATH = &#39;referencepath&#39;</pre><pre># FILE CATALOG CONSTANT<br>REFDATASETID = &#39;refdatasetid&#39;<br>REFDATASETPATH = &#39;refdatasetpath&#39;</pre><pre># LAST RUN DETAILS CONSTANT<br>LR_OFFSET = &#39;offset&#39;<br>LR_FEATUREID = &#39;featureid&#39;<br>LR_DATASETID = &#39;datasetid&#39;</pre><pre># CONSTANT<br>NONE = &#39;None&#39;<br>TRUE = &#39;True&#39;<br>DS_DATA_DATE = &#39;ds_data_date&#39;<br>RUN_ID_FORMAT = &#39;%Y%m%d%H%M%S&#39;<br>_ALL_ = &#39;_ALL_&#39;</pre><pre># MIRROR CONSTANTS<br>MIRROR_MODE = &quot;mirror_mode&quot;<br>MIRROR_PROP = &quot;mirror_prop&quot;</pre><pre>MIRRORMODE = &quot;mirrormode&quot;<br>MIRRORPROP = &quot;mirrorprop&quot;</pre><pre>DATE_FORMAT = &quot;%Y-%m-%d&quot;<br></pre><pre>class DbUtil():<br>    &quot;&quot;&quot;Database Entity class&quot;&quot;&quot;<br>    # DB QUERY MODES<br>    READ = &#39;read&#39;<br>    WRITE = &#39;write&#39;<br>    SGDS_READ = &#39;sgds_read&#39;<br>    SGDS_WRITE = &#39;sgds_write&#39;</pre><pre>    def __init__(self):<br>        try:<br>            config = ConfigReader([&#39;credentials.properties&#39;])<br>            db_config = config.get_db_config(DB_SECTION)<br>            self.db_conn = psycopg2.connect(&#39;&#39;)<br>            #Added below statement for EDB RW separation<br>            self.db_conn_write = psycopg2.connect(user=db_config[WRITE_USER],<br>                                                  password=db_config[WRITE_PASSWORD],<br>                                                  host=db_config[WRITE_HOST],<br>                                                  port=db_config[WRITE_PORT],<br>                                                  database=db_config[WRITE_DATABASE])<br>            #Added below statements for Sharding<br>            sgds_db_config = config.get_db_config(SGDS_DB_SECTION)<br>            self.sgds_db_conn = psycopg2.connect(user=sgds_db_config[READ_USER],<br>                                                 password=sgds_db_config[READ_PASSWORD],<br>                                                 host=sgds_db_config[READ_HOST],<br>                                                 port=sgds_db_config[READ_PORT],<br>                                                 database=sgds_db_config[READ_DATABASE])<br>            self.sgds_db_conn_write = psycopg2.connect(user=sgds_db_config[WRITE_USER],<br>                                                       password=sgds_db_config[WRITE_PASSWORD],<br>                                                       host=sgds_db_config[WRITE_HOST],<br>                                                       port=sgds_db_config[WRITE_PORT],<br>                                                       database=sgds_db_config[WRITE_DATABASE])<br>        except OperationalError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Error while connecting to database\n{message}&quot;.format(<br>                message=message))</pre><pre>    def __del__(self):<br>        try:<br>            self.db_conn.commit()<br>            self.db_conn.close()<br>            self.db_conn_write.commit()<br>            self.db_conn_write.close()<br>            self.sgds_db_conn.commit()<br>            self.sgds_db_conn.close()<br>            self.sgds_db_conn_write.commit()<br>            self.sgds_db_conn_write.close()<br>        except AttributeError:<br>            pass</pre><pre>    def excec_query(self, query, mode, fetch_all):<br>        &quot;&quot;&quot;Execute the query and return result set&quot;&quot;&quot;<br>        try:<br>            if mode == self.READ:<br>                cursor = self.db_conn.cursor(cursor_factory=RealDictCursor)<br>                cursor.execute(query)<br>                rows = None<br>                if fetch_all:<br>                    rows = cursor.fetchall()<br>                else:<br>                    rows = cursor.fetchone()<br>                return rows</pre><pre>            elif mode == self.WRITE:<br>                cursor = self.db_conn_write.cursor(cursor_factory=RealDictCursor)<br>                cursor.execute(query)</pre><pre>            if mode == self.SGDS_READ:<br>                cursor = self.sgds_db_conn.cursor(cursor_factory=RealDictCursor)<br>                cursor.execute(query)<br>                rows = None<br>                if fetch_all:<br>                    rows = cursor.fetchall()<br>                else:<br>                    rows = cursor.fetchone()<br>                return rows</pre><pre>            elif mode == self.SGDS_WRITE:<br>                cursor = self.sgds_db_conn_write.cursor(cursor_factory=RealDictCursor)<br>                cursor.execute(query)<br>        except OperationalError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Errors related to the database operation\n{message}&quot;.format(<br>                message=message))<br>        except IntegrityError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Relational integrity of the database is affected\n{message}&quot;.format(<br>                message=message))<br>        except ProgrammingError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Programming error: Table not found/SQL Syntax error\n{message}&quot;.format(message=message))<br>        except DatabaseError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Error while executing query\n{message}&quot;.format(message=message))</pre><pre>    def excec_many(self, query, arg_list):<br>        &quot;&quot;&quot;Execute the query and return result set&quot;&quot;&quot;<br>        try:<br>            cursor = self.db_conn_write.cursor(cursor_factory=RealDictCursor)<br>            rows = cursor.executemany(query, arg_list)</pre><pre>            return rows<br>        except OperationalError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Errors related to the database operation\n{message}&quot;.format(<br>                message=message))<br>        except IntegrityError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Relational integrity of the database is affected\n{message}&quot;.format(<br>                message=message))<br>        except ProgrammingError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Programming error: Table not found/SQL Syntax error\n{message}&quot; \<br>                              .format(message=message))<br>        except DatabaseError:<br>            message = get_error_message(sys.exc_info())<br>            raise DbException(&quot;Error while executing query\n{message}&quot;.format(message=message))</pre><pre>class S3Util():<br>    &quot;&quot;&quot;S3 Utility Class&quot;&quot;&quot;<br>    JSON = &#39;json&#39;<br>    CSV = &#39;csv&#39;<br>    PART = &#39;part&#39;<br>    TEXT = &#39;txt&#39;<br>    PARQUET = &#39;parquet&#39;<br>    PROPERTY = &#39;properties&#39;</pre><pre>    def __init__(self):<br>        self.s3_conn = boto3.resource(&#39;s3&#39;)</pre><pre>    def get_objects(self, bucket_name, object_path, object_type, part_flag):<br>        &quot;&quot;&quot;Get the required object from the  s3 bucket with prefix&quot;&quot;&quot;<br>        try:<br>            bucket_objects = self.s3_conn.meta.client.list_objects(Bucket=bucket_name,<br>                                                                   Prefix=object_path)<br>            files = []<br>            if bucket_objects:<br>                for obj in bucket_objects[&#39;Contents&#39;]:<br>                    file_type = &#39;.&#39; + object_type<br>                    file = str(obj.get(&#39;Key&#39;))<br>                    if part_flag:<br>                        if file_type in file and self.PART in file:<br>                            files.append(file)<br>                    else:<br>                        if file_type in file:<br>                            files.append(file)<br>            return files<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise S3Exception(&quot;Objects/path not found Bucket:{bucket} Path:{path}\n{message}&quot; \<br>                              .format(bucket=bucket_name, path=object_path, message=message))</pre><pre>    def get_object_data(self, bucket_name, object_path):<br>        &quot;&quot;&quot;Get content of the required object&quot;&quot;&quot;<br>        try:<br>            s3_object = None<br>            obj = self.s3_conn.Object(bucket_name, object_path)<br>            if obj:<br>                s3_object = obj.get()[&#39;Body&#39;].read().decode(&#39;utf-8&#39;)<br>            return s3_object<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))</pre><pre>    def get_all_s3_keys(self, bucket_name, object_path):<br>        &quot;&quot;&quot;Get all the s3 objects, when object count greater than 1000&quot;&quot;&quot;<br>        try:<br>            keys = []<br>            kwargs = {&#39;Bucket&#39;: bucket_name, &#39;Prefix&#39;: object_path}<br>            while True:<br>                resp = self.s3_conn.meta.client.list_objects_v2(**kwargs)<br>                for obj in resp[&#39;Contents&#39;]:<br>                    keys.append(obj[&#39;Key&#39;])<br>                try:<br>                    kwargs[&#39;ContinuationToken&#39;] = resp[&#39;NextContinuationToken&#39;]<br>                except KeyError:<br>                    break<br>            return keys<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise S3Exception(&quot;Objects/path not found Bucket:&#39;{bucket}&#39; Path:&#39;{path}&#39;\n{message}&quot; \<br>                              .format(bucket=bucket_name, path=object_path, message=message))<br>                              <br>    def get_all_s3_keys_if_available(self, bucket_name, object_path):<br>        &quot;&quot;&quot;Get all the s3 objects, when object count greater than 1000&quot;&quot;&quot;<br>        try:<br>            keys = []<br>            kwargs = {&#39;Bucket&#39;: bucket_name, &#39;Prefix&#39;: object_path}<br>            while True:<br>                resp = self.s3_conn.meta.client.list_objects_v2(**kwargs)<br>                for obj in resp[&#39;Contents&#39;]:<br>                    keys.append(obj[&#39;Key&#39;])<br>                try:<br>                    kwargs[&#39;ContinuationToken&#39;] = resp[&#39;NextContinuationToken&#39;]<br>                except KeyError:<br>                    break<br>            return keys<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                print(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            print(&quot;Objects/path not found Bucket:&#39;{bucket}&#39; Path:&#39;{path}&#39;\n{message}&quot; \<br>                              .format(bucket=bucket_name, path=object_path, message=message))</pre><pre>    def put_object(self, bucket_name, key, data):<br>        &quot;&quot;&quot;Upload data to s3 bucket&quot;&quot;&quot;<br>        try:<br>            self.s3_conn.Bucket(bucket_name).put_object(Key=key, Body=data)<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;Error while uploading to s3&quot;, str(client_error))<br>        except Exception as s3_exception:<br>            raise S3Exception(&quot;Error while uploading to s3&quot;, str(s3_exception))<br>            <br>    def move_object(self,bucket_name, source_key, destination_key):<br>        try:<br>            # Get the source and destination buckets<br>            print(&quot;step-1&quot;)<br>            source_bucket = self.s3_conn.Bucket(bucket_name)<br>            print(&quot;step-2&quot;)<br>            destination_bucket = self.s3_conn.Bucket(bucket_name)<br>            print(&quot;step-3&quot;)<br>            source_obj = self.s3_conn.Object(bucket_name, source_key)<br>            print(&quot;step-4&quot;)<br>            destination_obj = destination_bucket.Object(destination_key)<br>            print(&quot;step-5&quot;)<br>            destination_obj.copy({&#39;Bucket&#39;: bucket_name, &#39;Key&#39;: source_key})<br>            print(&quot;step-6&quot;)</pre><pre>            # Delete the object from the source<br>            source_obj.delete()<br>            #self.s3_conn.copy_object(Bucket=bucket_name, CopySource={&#39;Bucket&#39;: bucket_name, &#39;Key&#39;: source_key}, Key=destination_key)<br>            #self.s3_conn.delete_object(Bucket=bucket_name, Key=source_key)<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;Error while uploading to s3&quot;, str(client_error))<br>        except Exception as s3_exception:<br>            raise S3Exception(&quot;Error while uploading to s3&quot;, str(s3_exception))</pre><pre>    def delete_object(self, bucket_name, key):<br>        &quot;&quot;&quot; Delete s3 object from the bucket&quot;&quot;&quot;<br>        try:<br>            self.s3_conn.Object(bucket_name, key).load()<br>            self.s3_conn.Object(bucket_name, key).delete()<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=key, message=message))<br>        except Exception as s3_exception:<br>            raise S3Exception(&quot;Error while deleting from s3&quot;, str(s3_exception))<br>            <br>    def download(self, bucket_name, object_path, destination_file):<br>        &quot;&quot;&quot;Get the required object from the  s3 bucket with prefix&quot;&quot;&quot;<br>        try:<br>            self.s3_conn.meta.client.download_file(bucket_name,object_path,destination_file)</pre><pre>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise S3Exception(&quot;Objects/path not found Bucket:{bucket} Path:{path}\n{message}&quot; \<br>                              .format(bucket=bucket_name, path=object_path, message=message))<br>    def upload(self, file_name, bucket_name, object_path):<br>        &quot;&quot;&quot;Get the required object from the  s3 bucket with prefix&quot;&quot;&quot;<br>        try:<br>            response = self.s3_conn.meta.client.upload_file(file_name, bucket_name, object_path)<br>            return response<br>        except ClientError as client_error:<br>            message = get_error_message(sys.exc_info())<br>            if client_error.response[&#39;Error&#39;][&#39;Code&#39;] == &#39;NoSuchBucket&#39;:<br>                raise S3Exception(&quot;Bucket &#39;{bucket}&#39; not found \n{message}&quot;.format(<br>                    bucket=bucket_name, message=message))<br>            else:<br>                raise S3Exception(&quot;File &#39;{path}&#39; not found in bucket &#39;{bucket}&#39;\n{message}&quot;.format(<br>                    bucket=bucket_name, path=object_path, message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise S3Exception(&quot;Objects/path not found Bucket:{bucket} Path:{path}\n{message}&quot; \<br>                              .format(bucket=bucket_name, path=object_path, message=message))<br></pre><pre>class ConfigReader():<br>    &quot;&quot;&quot;Configuration Reader Class&quot;&quot;&quot;</pre><pre>    def __init__(self, config_file):<br>        self.config_file = config_file<br>        self.config_reader = self.get_config_reader()</pre><pre>    def get_config_reader(self):<br>        &quot;&quot;&quot;Read configuration file&quot;&quot;&quot;<br>        config = configparser.RawConfigParser()<br>        config.read(self.config_file)<br>        return config</pre><pre>    def get_s3_config(self):<br>        &quot;&quot;&quot;Get S3 Configuration from configuration file&quot;&quot;&quot;<br>        try:<br>            access_id = self.config_reader.get(S3_SECTION, ACCESS_ID)<br>            access_key = self.config_reader.get(S3_SECTION, ACCESS_KEY)<br>            return {ACCESSID: access_id,<br>                    ACCESSKEY: access_key}<br>        except NoSectionError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Section not found in Configuration File\n{message}&quot;.format(<br>                message=message))<br>        except NoOptionError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Key not found in Configuration File\n{message}&quot;.format(<br>                message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Key not found\n{message}&quot;.format(message=message))</pre><pre>    def get_db_config(self, dbsection):<br>        &quot;&quot;&quot;Get Database configuration from configuration file&quot;&quot;&quot;<br>        try:<br>            read_host = self.config_reader.get(dbsection, READ_DB_HOST)<br>            read_port = self.config_reader.get(dbsection, READ_DB_PORT)<br>            read_user = self.config_reader.get(dbsection, READ_DB_USER)<br>            read_password = self.config_reader.get(dbsection, READ_DB_PASSWORD)<br>            read_database = self.config_reader.get(dbsection, READ_DB_DATABASE)<br>            read_options = self.config_reader.get(dbsection, READ_PG_OPTION)</pre><pre>            write_host = self.config_reader.get(dbsection, WRITE_DB_HOST)<br>            write_port = self.config_reader.get(dbsection, WRITE_DB_PORT)<br>            write_user = self.config_reader.get(dbsection, WRITE_DB_USER)<br>            write_password = self.config_reader.get(dbsection, WRITE_DB_PASSWORD)<br>            write_database = self.config_reader.get(dbsection, WRITE_DB_DATABASE)<br>            write_options = self.config_reader.get(dbsection, WRITE_PG_OPTION)<br>            #Added new attributes in return for EDB RW separation<br>            return {READ_HOST: read_host, READ_PORT: read_port, READ_USER: read_user, READ_PASSWORD: read_password, READ_DATABASE: read_database, READ_PG_OPTION: read_options, WRITE_HOST: write_host, WRITE_PORT: write_port, WRITE_USER: write_user, WRITE_PASSWORD: write_password, WRITE_DATABASE: write_database, WRITE_PG_OPTION: write_options}<br>        except NoSectionError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Section not found in Configuration File\n{message}&quot;.format(<br>                message=message))<br>        except NoOptionError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Key not found in Configuration File\n{message}&quot;.format(<br>                message=message))<br>        except KeyError:<br>            message = get_error_message(sys.exc_info())<br>            raise ConfigException(&quot;Key not found\n{message}&quot;.format(message=message))<br></pre><pre>class InputConfig():<br>    &quot;&quot;&quot;Input Configuration Class&quot;&quot;&quot;</pre><pre>    def __init__(self, run_id, prop_code, job_id):<br>        self.run_id = run_id<br>        self.prop_code = prop_code<br>        self.job_id = job_id</pre><pre>    def get_kdate(self, db_util):<br>        &quot;&quot;&quot;Get K date for the property&quot;&quot;&quot;<br>        try:<br>            query = &quot;&quot;&quot;SELECT ds_data_date from {schema}.rms_data_status<br>            WHERE ds_prop_code = &#39;{prop_code}&#39; AND ds_process_action = &#39;K_DATE&#39;<br>            and ds_active&quot;&quot;&quot;.format(schema=os.environ[SCHEMA], prop_code=self.prop_code)</pre><pre>            row = db_util.excec_query(query, db_util.READ, False)<br>            return row[&#39;ds_data_date&#39;]</pre><pre>        except Exception:<br>            message = get_error_message(sys.exc_info())<br>            print(message)<br>            raise DbException(&quot;K date not found&quot;)</pre><pre>    def get_mirror_info(self, db_util):<br>        &quot;&quot;&quot;Get Mirror details&quot;&quot;&quot;<br>        try:<br>            query = &quot;&quot;&quot;SELECT ppa_param_value AS mirror_mode<br>            FROM {schema}.prop_param_assign WHERE ppa_param_id = &#39;MIRROR_MODE&#39;<br>            AND ppa_prop_code = &#39;{prop_code}&#39;&quot;&quot;&quot;.format(schema=os.environ[SCHEMA],<br>                                                        prop_code=self.prop_code)</pre><pre>            row = db_util.excec_query(query, db_util.READ, False)<br>            mirror_mode = int(row[MIRROR_MODE])<br>            if mirror_mode == 2:<br>                query = &quot;&quot;&quot;SELECT ppa_param_value AS mirror_prop<br>                FROM {schema}.prop_param_assign where ppa_param_id = &#39;MIRPROP&#39;<br>                AND ppa_prop_code = &#39;{prop_code}&#39;&quot;&quot;&quot;.format(schema=os.environ[SCHEMA],<br>                                                            prop_code=self.prop_code)<br>                row = db_util.excec_query(query, db_util.READ, False)<br>                mirror_prop = row[MIRROR_PROP]<br>            else:<br>                mirror_prop = &quot;null&quot;</pre><pre>            return {MIRRORMODE: mirror_mode,<br>                    MIRRORPROP: mirror_prop<br>                    }<br>        except Exception:<br>            message = get_error_message(sys.exc_info())<br>            print(message)<br>            raise Exception(&quot;Mirror property information not found.&quot;)</pre><pre>    def get_input_ds(self, run_id, datasets, run_date, db_util):<br>        &quot;&quot;&quot;Get all the dataset list from config&quot;&quot;&quot;<br>        try:<br>            input_data_path = []</pre><pre>            for dataset in datasets[DS_DATASETS]:<br>                dataset_id = dataset[DS_DATASETID]<br>                if dataset[DS_BYPROPERTY] != &#39;True&#39;:<br>                    sd_prop_code = &#39;_ALL_&#39;<br>                else:<br>                    sd_prop_code = self.prop_code</pre><pre>                query = &quot;&quot;&quot;SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date, sd_data_date_from,<br>                sd_data_date_to, sd_warning_date, sd_valid_from, sd_valid_to, sd_asof_date, sd_applicable_date_from, <br>                sd_applicable_date_to FROM {schema}.rms_shared_dataset<br>                WHERE sd_feature_id = &#39;{sd_feature_id}&#39;<br>                AND sd_dataset_id = &#39;{sd_dataset_id}&#39;<br>                --AND sd_run_id = &#39;{sd_run_id}&#39;<br>                AND sd_entity = &#39;{sd_entity}&#39;<br>                &quot;&quot;&quot;</pre><pre>                if dataset[DS_CONDITION] != NONE:<br>                    query = query + dataset[DS_CONDITION]</pre><pre>                query = query.format(schema=os.environ[SCHEMA], sd_feature_id=dataset[DS_FEATUREID],<br>                                     sd_dataset_id=dataset_id, sd_run_id=run_id,<br>                                     sd_entity=sd_prop_code, sd_run_date=run_date)<br>                # print(query)</pre><pre>                rows = db_util.excec_query(query, DbUtil.READ, True)<br>                # print(rows)<br>                if rows:<br>                    input_data = {DATASETID: dataset_id,<br>                                  RUNID: rows[0][SD_RUN_ID],<br>                                  DATAFROMDATE: str(rows[0][SD_DATA_DATE_FROM]),<br>                                  DATATODATE: str(rows[0][SD_DATA_DATE_TO]),<br>                                  VALIDFROM: str(rows[0][SD_VALID_FROM]),<br>                                  VALIDTO: str(rows[0][SD_VALID_TO]),<br>                                  WARNINGDATE: str(rows[0][SD_WARNING_DATE]),<br>                                  ASOFDATE: str(rows[0][SD_ASOF_DATE]),<br>                                  SDSK: rows[0][SD_SK],<br>                                  DATASETPATH: rows[0][SD_DATASET_PATH]}<br>                else:<br>                    input_data = {DATASETID: dataset_id,<br>                                  ASOFDATE: &quot;&quot;,<br>                                  RUNID: -1,<br>                                  SDSK: 0,<br>                                  DATASETPATH: &quot;&quot;}</pre><pre>                input_data_path.append(input_data)<br>            return input_data_path<br>        except KeyError as key_error:<br>            message = get_error_message(sys.exc_info())<br>            print(message)<br>            raise ConfigException(&quot;Key {key} not found in &#39;{option}&#39; of jobconfig&quot;.format(<br>                option=INPUT_DS_LIST, key=str(key_error)))</pre><pre>    def gen_input_ds(self, run_id, datasets, run_date, db_util):<br>        &quot;&quot;&quot;Get all the dataset list from config&quot;&quot;&quot;<br>        try:<br>            input_data_path = []<br>            schema = os.environ[SCHEMA]<br>            for dataset in datasets[DS_DATASETS]:<br>                dataset_id = dataset[DS_DATASETID]<br>                if dataset[DS_BYPROPERTY] != &#39;True&#39;:<br>                    sd_prop_code = &#39;_ALL_&#39;<br>                else:<br>                    sd_prop_code = self.prop_code</pre><pre>                query = &quot;&quot;&quot;SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date, sd_data_date_from,<br>                sd_data_date_to, sd_warning_date, sd_valid_from, sd_valid_to, sd_asof_date, sd_applicable_date_from, <br>                sd_applicable_date_to FROM {schema}.rms_shared_dataset<br>                WHERE sd_feature_id = &#39;{sd_feature_id}&#39;<br>                AND sd_dataset_id = &#39;{sd_dataset_id}&#39;<br>                --AND sd_run_id = &#39;{sd_run_id}&#39;<br>                AND sd_entity = &#39;{sd_entity}&#39;<br>                &quot;&quot;&quot;</pre><pre>                if dataset[DS_CONDITION] != NONE:<br>                    query = query + dataset[DS_CONDITION]</pre><pre>                query = query.format(schema=schema, sd_feature_id=dataset[DS_FEATUREID],<br>                                     sd_dataset_id=dataset_id, sd_run_id=run_id, sd_entity=sd_prop_code,<br>                                     sd_run_date=run_date)<br>                # print(query)</pre><pre>                rows = db_util.excec_query(query, DbUtil.READ, True)<br>                if not rows:<br>                    raise DbException(<br>                        &quot;Dataset Id &#39;{dataset_id}&#39; not found for property &#39;{prop_code}&#39; \nRMS SHARED DATASET Query = {query}&quot;<br>                        &quot;&quot;.format(dataset_id=dataset_id, prop_code=sd_prop_code, query=query))<br>                # print(rows)<br>                input_data = {DATASETID: dataset_id,<br>                              RUNID: rows[0][SD_RUN_ID],<br>                              DATAFROMDATE: str(rows[0][SD_DATA_DATE_FROM]),<br>                              DATATODATE: str(rows[0][SD_DATA_DATE_TO]),<br>                              VALIDFROM: str(rows[0][SD_VALID_FROM]),<br>                              VALIDTO: str(rows[0][SD_VALID_TO]),<br>                              WARNINGDATE: str(rows[0][SD_WARNING_DATE]),<br>                              ASOFDATE: str(rows[0][SD_ASOF_DATE])}<br>                if len(rows) &gt; 1:<br>                    sdSk = []<br>                    datasetPath = []<br>                    for row in rows:<br>                        sdSk.append(str(row[SD_SK]))<br>                        datasetPath.append(row[SD_DATASET_PATH])<br>                    input_data[SDSK] = COMMA_DELIM.join(sdSk)<br>                    input_data[DATASETPATH] = COMMA_DELIM.join(datasetPath)<br>                else:<br>                    input_data[SDSK] = rows[0][SD_SK]<br>                    input_data[DATASETPATH] = rows[0][SD_DATASET_PATH]</pre><pre>                input_data_path.append(input_data)<br>            return input_data_path<br>        except KeyError as key_error:<br>            message = get_error_message(sys.exc_info())<br>            print(message)<br>            raise ConfigException(&quot;Key {key} not found in &#39;{option}&#39; of jobconfig&quot;.format(<br>                option=INPUT_DS_LIST, key=str(key_error)))</pre><pre>    def gen_input_datasets(self, datasets, asof_date, data_date, db_util):<br>        &quot;&quot;&quot;Get all the dataset list from config&quot;&quot;&quot;<br>        try:<br>            input_data_path = []</pre><pre>            for dataset in datasets[DS_DATASETS]:<br>                dataset_id = dataset[DS_DATASETID]<br>                if dataset[DS_BYPROPERTY] != &#39;True&#39;:<br>                    prop_code = &#39;_ALL_&#39;<br>                else:<br>                    prop_code = self.prop_code<br>                try:<br>                    is_mandate = dataset[DS_ISMANDATE]</pre><pre>                    if is_mandate.capitalize() == &#39;True&#39;:<br>                        is_mandate = True<br>                    else:<br>                        is_mandate = False<br>                except KeyError:<br>                    is_mandate = False</pre><pre>                query = &quot;&quot;&quot;SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date,<br>                sd_data_date_from, sd_data_date_to, sd_warning_date,<br>                sd_valid_from, sd_valid_to, sd_asof_date,<br>                sd_applicable_date_from, sd_applicable_date_to<br>                FROM {schema}.rms_shared_dataset WHERE sd_feature_id = &#39;{feature_id}&#39;<br>                AND sd_dataset_id = &#39;{dataset_id}&#39;<br>                AND sd_entity = &#39;{prop_code}&#39; &quot;&quot;&quot;</pre><pre>                if dataset[DS_CONDITION] != NONE:<br>                    query = query + dataset[DS_CONDITION]</pre><pre>                query = query.format(schema=os.environ[SCHEMA], feature_id=dataset[DS_FEATUREID],<br>                                     dataset_id=dataset_id, run_id=self.run_id,<br>                                     prop_code=prop_code, asof_date=asof_date, data_date=data_date)<br>                #print(query)</pre><pre>                row = db_util.excec_query(query, DbUtil.READ, False)<br>                if not row and is_mandate:<br>                    raise DbException(&quot;Dataset Id &#39;{dataset_id}&#39; not found for property \<br>&#39;{prop_code}&#39; \nRMS SHARED DATASET Query = {query}&quot;.format(dataset_id=dataset_id,<br>                                                           prop_code=prop_code, query=query))<br>                #print(row)<br>                if row:<br>                    input_data = {<br>                        SDSK: row[SD_SK],<br>                        DATASETID: dataset_id,<br>                        RUNID: row[SD_RUN_ID],<br>                        ASOFDATE: str(row[SD_ASOF_DATE]),<br>                        DATADATEFROM: str(row[SD_DATA_DATE_FROM]),<br>                        DATADATETO: str(row[SD_DATA_DATE_TO]),<br>                        VALIDFROM: str(row[SD_VALID_FROM]),<br>                        VALIDTO: str(row[SD_VALID_TO]),<br>                        WARNINGDATE: str(row[SD_WARNING_DATE]),<br>                        DATASETPATH: row[SD_DATASET_PATH]<br>                    }<br>                else:<br>                    input_data = {<br>                        ASOFDATE: &quot;&quot;,<br>                        DATASETID: dataset_id,<br>                        RUNID: -1,<br>                        SDSK: 0,<br>                        DATASETPATH: &quot;&quot;<br>                    }<br>                input_data_path.append(input_data)<br>            return input_data_path<br>        except KeyError as key_error:<br>            # message = get_error_message(sys.exc_info())<br>            print(get_error_message(sys.exc_info()))<br>            raise ConfigException(&quot;Key {key} not found in &#39;{option}&#39; of jobconfig&quot;.format(<br>                option=INPUT_DS_LIST, key=str(key_error)))</pre><pre>    def get_reference_data(self):<br>        &quot;&quot;&quot;Generate reference brand &amp; control file json data&quot;&quot;&quot;<br>        catalog_file_path = os.environ[FILECATALOG]<br>        bucket_name = os.environ[BUCKETNAME]<br>        s3_util = S3Util()<br>        file_catalog = json.loads(s3_util.get_object_data(bucket_name, catalog_file_path))</pre><pre>        try:<br>            ref_datasets = file_catalog[self.job_id]<br>        except KeyError:<br>            raise ConfigException(&quot;No reference files found in File Catalog for &#39;{job_id}&#39;&quot;. \<br>                                  format(job_id=self.job_id))<br>        reference_data_path = []<br>        try:<br>            for ref_dataset in ref_datasets:<br>                reference_data = {REFERENCEFILEID: ref_dataset[REFDATASETID],<br>                                  REFERENCEPATH: ref_dataset[REFDATASETPATH]}<br>                reference_data_path.append(reference_data)<br>        except KeyError as key_error:<br>            raise ConfigException(&quot;No key {key_error} in &#39;{job_id}&#39; of File_catalog&quot;.format(<br>                key_error=str(key_error), job_id=self.job_id))<br>        return reference_data_path</pre><pre>    def get_last_run_details(self, last_run_details, db_util, last_run_field=&#39;sd_run_date&#39;):<br>        &quot;&quot;&quot;Get last successful run of the spark using dataset id&quot;&quot;&quot;<br>        try:<br>            sd_dataset_id = last_run_details[LR_DATASETID]<br>            sd_feature_id = last_run_details[LR_FEATUREID]<br>            offset = last_run_details[LR_OFFSET]<br>        except KeyError as key_error:<br>            raise ConfigException(&quot;Key {key} not found in &#39;last_run_details&#39; of jobconfig&quot;.format(<br>                key=str(key_error)))<br>        schema = os.environ[SCHEMA]<br>        query = &quot;&quot;&quot;SELECT ({last_run_field} - INTERVAL &#39;{days} DAY&#39;)::DATE AS sd_run_date<br>        FROM {schema}.rms_shared_dataset<br>        WHERE sd_feature_id = &#39;{sd_feature_id}&#39;<br>        AND sd_dataset_id = &#39;{sd_dataset_id}&#39;<br>        AND sd_entity = &#39;{sd_entity}&#39;<br>        ORDER BY sd_run_id DESC LIMIT 1<br>        &quot;&quot;&quot;.format(days=offset, schema=schema, sd_feature_id=sd_feature_id,<br>                   sd_dataset_id=sd_dataset_id, sd_entity=self.prop_code,<br>                   last_run_field=last_run_field)</pre><pre>        row = db_util.excec_query(query, DbUtil.READ, False)<br>        if not row:<br>            raise DbException(&quot;No entry found &#39;{sd_dataset_id}&#39; for &#39;property &#39;{prop_code}&#39;&quot;.<br>                              format(sd_dataset_id=sd_dataset_id, prop_code=self.prop_code))<br>        return row[SD_RUN_DATE]</pre><pre>    def get_rms_data_status_data(self, ds_feature, db_util):<br>        &quot;&quot;&quot;Get as of date for the flow&quot;&quot;&quot;<br>        schema = os.environ[SCHEMA]<br>        query = &quot;&quot;&quot;SELECT ds_data_date<br>        FROM {schema}.rms_data_status<br>        WHERE ds_process_action = &#39;{ds_process_action}&#39;<br>        AND ds_prop_code = &#39;{ds_prop_code}&#39;<br>        &quot;&quot;&quot;.format(schema=schema, ds_process_action=ds_feature, ds_prop_code=self.prop_code)<br>        row = db_util.excec_query(query, DbUtil.READ, False)<br>        if not row:<br>            raise DbException(&quot;No &#39;ds_data_date&#39; found for property &#39;{prop_code}&#39;&quot;.format(<br>                prop_code=self.prop_code))<br>        return row[&#39;ds_data_date&#39;]</pre><pre>    def get_run_id(self):<br>        &quot;&quot;&quot;return run_id&quot;&quot;&quot;<br>        return self.run_id</pre><pre>    def get_prop_code(self):<br>        &quot;&quot;&quot;return prop_code&quot;&quot;&quot;<br>        return self.prop_code</pre><pre>    def get_job_id(self):<br>        &quot;&quot;&quot;return job_id&quot;&quot;&quot;<br>        return self.job_id<br></pre><pre>def parse_s3_path(path):<br>    &#39;&#39;&#39;Parse S3 Paths&#39;&#39;&#39;<br>    try:<br>        obj = urlparse(path)<br>        return obj.netloc, obj.path.lstrip(&#39;/&#39;)<br>    except Exception as url_parse_exception:<br>        raise UrlParseException(&quot;Error while parsing s3 path&quot;, str(url_parse_exception))<br></pre><pre>def get_latest_file(bucket_name, file_path, file_type, part_flag, s3_util):<br>    &quot;&quot;&quot;Get latest rms json from outputjsonpath of input json&quot;&quot;&quot;<br>    if part_flag:<br>        file_path = file_path + &#39;part&#39;<br>    file_type = &#39;.&#39; + file_type</pre><pre>    response = s3_util.s3_conn.meta.client.list_objects_v2(<br>        Bucket=bucket_name, Prefix=file_path)<br>    file_list = list(filter(lambda file_info: file_type in file_info[&#39;Key&#39;], response[&#39;Contents&#39;]))<br>    latest = max(file_list, key=lambda x: x[&#39;LastModified&#39;])<br>    return latest[&#39;Key&#39;]<br></pre><pre>def get_opts(arguments):<br>    &quot;&quot;&quot;Read command line arguments&quot;&quot;&quot;<br>    opts = {}  # Empty dictionary to store key-value pairs.<br>    while arguments:  # While there are arguments left to parse...<br>        if arguments[0][0] == &#39;-&#39;:  # Found a &quot;-name value&quot; pair.<br>            opts[arguments[0]] = arguments[1]  # Add key and value to the dictionary.<br>        arguments = arguments[1:]  # Reduce argument list by copying it starting from index 1.<br>    return opts<br></pre><pre>def get_error_message(exception_info):<br>    &quot;&quot;&quot;Generate the error message&quot;&quot;&quot;<br>    exc_type, exc_obj, exc_tb = exception_info<br>    error_message = &quot;&quot;&quot;Error Class: {type}\nMessage: {obj}\nFile: {file}\nLine No: {line}&quot;&quot;&quot; \<br>        .format(type=exc_type, obj=exc_obj,<br>                file=os.path.split(exc_tb.tb_frame.f_code.co_filename)[-1],<br>                line=exc_tb.tb_lineno)<br>    return error_message<br></pre><pre>def get_certificate_name():<br>    &quot;&quot;&quot;Returns certificate name&quot;&quot;&quot;<br>    cred_config = ConfigReader([&#39;credentials.properties&#39;])<br>    conf = cred_config.get_config_reader()</pre><pre>    return str(conf.get(ENV_SECTION, &#39;CERTIFICATE_NAME&#39;))<br></pre><pre>def set_env_variables(cred_file):<br>    &quot;&quot;&quot;Return the database schema&quot;&quot;&quot;<br>    try:<br>        config = ConfigReader(cred_file)<br>        db_config = config.get_db_config(DB_SECTION)<br>        #s3_config = config.get_s3_config()</pre><pre>        # Setting PostgreSQL Environment variables for PSYCOPG2<br>        os.environ[PGHOST] = db_config[READ_HOST]<br>        os.environ[PGPORT] = db_config[READ_PORT]<br>        os.environ[PGUSER] = db_config[READ_USER]<br>        os.environ[PGPASSWORD] = db_config[READ_PASSWORD]<br>        os.environ[PGDATABASE] = db_config[READ_DATABASE]<br>        #os.environ[PGOPTIONS] = db_config[READ_PG_OPTION]</pre><pre>        # Setting S3 Environment variables for BOTO3<br>        #os.environ[AWS_ACCESS_KEY_ID] = s3_config[ACCESSID]<br>        #os.environ[AWS_SECRET_ACCESS_KEY] = s3_config[ACCESSKEY]</pre><pre>        # Setting Environment variable for schema and bucket<br>        config_reader = config.get_config_reader()<br>        os.environ[SCHEMA] = config_reader.get(ENV_SECTION, DB_SCHEMA)<br>        os.environ[BUCKETNAME] = config_reader.get(ENV_SECTION, BUCKET_NAME)<br>        os.environ[FILECATALOG] = config_reader.get(ENV_SECTION, FILE_CATALOG)<br>        # os.environ[REQUESTS_CA_BUNDLE] = config_reader.get(ENV_SECTION, &#39;SSL_CERTIFICATE_PATH&#39;)<br>    except NoSectionError:<br>        message = get_error_message(sys.exc_info())<br>        print(&quot;ossection&quot;)<br>        raise ConfigException(&quot;Section not found in Configuration File\n{message}&quot;.format(<br>            message=message))<br>    except NoOptionError:<br>        message = get_error_message(sys.exc_info())<br>        raise ConfigException(&quot;Key not found in Configuration File\n{message}&quot;.format(<br>            message=message))<br>    except KeyError:<br>        message = get_error_message(sys.exc_info())<br>        raise ConfigException(&quot;Key not found\n{message}&quot;.format(message=message))<br>    except ConfigException:<br>        message = get_error_message(sys.exc_info())<br>        raise EnvironmentException(&quot;Error while setting environment variables\n{message}&quot;.format(<br>            message=message))<br>    except Exception:<br>        message = get_error_message(sys.exc_info())<br>        raise EnvironmentException(&quot;Error while setting environment variables\n{message}&quot;.format(<br>            message=message))<br></pre><pre>def string_to_date(date_string, date_format):<br>    &quot;&quot;&quot;Convert string to desired date format&quot;&quot;&quot;<br>    return datetime.strptime(date_string, date_format).date()<br></pre><pre>def rest_call(endpoint, arg_list, method=&#39;GET&#39;):<br>    config_util = ConfigReader([&#39;credentials.properties&#39;])<br>    data = &quot;&quot;<br>    res = &#39;&#39;<br>    &quot;&quot;&quot;REST SERVICE Call&quot;&quot;&quot;<br>    if method == &#39;GET&#39;:<br>        arg_vals = arg_list.items()<br>        url = endpoint + HOST_DELIM<br>        for arg in arg_vals:<br>            url = url + arg[0] + EQUALSTO + arg[1] + QUERY_DELIM<br>        url = url.rstrip(QUERY_DELIM)<br>        # print(url)</pre><pre>        if &#39;https&#39; in url:<br>            res = requests.post(url, json=data, verify=get_certificate_name())<br>            # res = requests.post(url, json=data, verify=False)</pre><pre>        else:<br>            res = requests.post(url, json=data)</pre><pre>    if method == &#39;POST&#39;:<br>        data = json.loads(json.dumps(arg_list, cls=Encoder))<br>        url = endpoint<br>        if &#39;https&#39; in url:<br>            res = requests.post(url, json=data, verify=get_certificate_name())<br>            # res = requests.post(url, json=data, verify=False)<br>        else:<br>            res = requests.post(url, json=data)</pre><pre>    if method == &#39;PUT&#39;:<br>        data = json.loads(json.dumps(arg_list, cls=Encoder))<br>        url = endpoint<br>        if &#39;https&#39; in url:<br>            res = requests.put(url, json=data, verify=get_certificate_name())<br>            # res = requests.put(url, json=data, verify=False)<br>        else:<br>            res = requests.put(url, json=data)<br>    return res<br></pre><pre>class HTTPAdapterWithSocketOptions(requests.adapters.HTTPAdapter):<br>    def __init__(self, *args, **kwargs):<br>        self.socket_options = kwargs.pop(&quot;socket_options&quot;, None)<br>        super(HTTPAdapterWithSocketOptions, self).__init__(*args, **kwargs)</pre><pre>    def init_poolmanager(self, *args, **kwargs):<br>        if self.socket_options is not None:<br>            kwargs[&quot;socket_options&quot;] = self.socket_options<br>            super(HTTPAdapterWithSocketOptions, self).init_poolmanager(*args, **kwargs)<br></pre><pre>def run_service(method=None, url=None, headers=None, files=None, data=None,<br>                params=None, auth=None, cookies=None, hooks=None, json=None,<br>                verify=False, retry_total=0, timeout=None, status_forcelist=[502, 503, 504],<br>                method_whitelist=[&quot;GET&quot;, &quot;POST&quot;, &quot;PUT&quot;], backoff_factor=0):<br>    if method is None or url is None:<br>        return None<br>    socket_options = urllib3.connection.HTTPConnection.default_socket_options + [<br>        (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),<br>        (socket.SOL_TCP, socket.TCP_KEEPINTVL, 180),<br>        (socket.SOL_TCP, socket.TCP_KEEPIDLE, 60),<br>        (socket.SOL_TCP, socket.TCP_KEEPCNT, 127)]<br>    retry_strategy = urllib3.util.retry.Retry(total=retry_total, status_forcelist=status_forcelist, backoff_factor=backoff_factor,<br>                                              method_whitelist=method_whitelist)<br>    retry_strategy.BACKOFF_MAX = 300<br>    adapter = HTTPAdapterWithSocketOptions(max_retries=retry_strategy, socket_options=socket_options)<br>    with requests.session() as s:<br>        s.mount(&quot;http://&quot;, adapter)<br>        s.mount(&quot;https://&quot;, adapter)<br>        req = requests.Request(method, url, headers, files, data, params, auth, cookies, hooks, json)<br>        prepped = s.prepare_request(req)<br>        with s.send(prepped, verify=verify, timeout=timeout) as r:<br>            logging.info(r)<br>            return r<br></pre><pre>class Encoder(json.JSONEncoder):<br>    &quot;&quot;&quot;Encoder for Json dump&quot;&quot;&quot;</pre><pre>    def default(self, o):<br>        if isinstance(o, date):<br>            return o.isoformat()<br>        elif isinstance(o, decimal.Decimal):<br>            return str(o)<br>        return json.JSONEncoder.default(self, o)<br></pre><pre>class ConfigException(Exception):<br>    &quot;&quot;&quot;Exception class Configuration Reader&quot;&quot;&quot;<br>    pass<br></pre><pre>class DbException(Exception):<br>    &quot;&quot;&quot;Exception class for Database&quot;&quot;&quot;<br>    pass<br></pre><pre>class EnvironmentException(Exception):<br>    &quot;&quot;&quot;Exception class for Environment Variable error&quot;&quot;&quot;<br>    pass<br></pre><pre>class S3Exception(Exception):<br>    &quot;&quot;&quot;Exception class for S3&quot;&quot;&quot;<br>    pass<br></pre><pre>class JsonExeption(Exception):<br>    &quot;&quot;&quot;Exception class for JSON realted errors&quot;&quot;&quot;<br>    pass<br></pre><pre>class HttpException(Exception):<br>    &quot;&quot;&quot;Exception class for HTTP realted errors&quot;&quot;&quot;<br>    pass<br></pre><pre>class UrlParseException(Exception):<br>    &quot;&quot;&quot;Exception class for S3 path parsing related errors&quot;&quot;&quot;<br>    pass<br></pre><pre>class FormatException(Exception):<br>    &quot;&quot;&quot;Exception class for Format exception&quot;&quot;&quot;<br>    pass<br></pre><pre>class EmailException(Exception):<br>    &quot;&quot;&quot;Exception class for Format exception&quot;&quot;&quot;<br>    pass<br></pre><pre>class OozieException(Exception):<br>    &quot;&quot;&quot;Exception class for Oozie exception&quot;&quot;&quot;<br>    pass</pre><p>dbconfig</p><pre>dbconfig<br>{<br>        jdbcconfig={<br>                default {<br>                   jdbcurl = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>                   drivername = &quot;org.postgresql.Driver&quot;<br>                   username = &quot;appuser&quot;<br>                   password = &quot;JbjC5nWjQR0nZDvS&quot;<br>                   poolsize = 5<br>                   idletimeout = 120000<br>                   querytimeout=&quot;50000 millis&quot;<br>                   jdbcurlro = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>                   drivernamero = &quot;org.postgresql.Driver&quot;<br>                   usernamero = &quot;appuser&quot;<br>                   passwordro = &quot;JbjC5nWjQR0nZDvS&quot;<br>                   poolsizero = 5<br>                   idletimeoutro = 120000<br>                   querytimeoutro=&quot;50000 millis&quot;<br>                   jdbcurlwo = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;<br>                   drivernamewo = &quot;org.postgresql.Driver&quot;<br>                   usernamewo = &quot;appuser&quot;<br>                   passwordwo = &quot;JbjC5nWjQR0nZDvS&quot;<br>                   poolsizewo = 5<br>                   idletimeoutwo = 120000<br>                   querytimeoutwo=&quot;50000 millis&quot;<br>                   databasename=&quot;ymdb&quot;<br>                }<br>    rpo{<br>       jdbcurl = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rpo-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_rpo?prepareThreshold=0&quot;<br>                   drivername = &quot;org.postgresql.Driver&quot;<br>                   username = &quot;appuser&quot;<br>                   password = &quot;0a8BKpJTUP84KsTt&quot;<br>                   poolsize = 5<br>                   idletimeout = 120000<br>                   querytimeout=&quot;50000 millis&quot;<br>                   jdbcurlro = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rpo-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_rpo?prepareThreshold=0&quot;<br>                   drivernamero = &quot;org.postgresql.Driver&quot;<br>                   usernamero = &quot;appuser&quot;<br>                   passwordro = &quot;0a8BKpJTUP84KsTt&quot;<br>                   poolsizero = 5<br>                   idletimeoutro = 120000<br>                   querytimeoutro=&quot;50000 millis&quot;<br>                   jdbcurlwo = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rpo-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_rpo?prepareThreshold=0&quot;<br>                   drivernamewo = &quot;org.postgresql.Driver&quot;<br>                   usernamewo = &quot;appuser&quot;<br>                   passwordwo = &quot;0a8BKpJTUP84KsTt&quot;<br>                   poolsizewo = 5<br>                   idletimeoutwo = 120000<br>                   querytimeoutwo=&quot;50000 millis&quot;<br>                   databasename=&quot;ymdb_rpo&quot;<br>    }<br>    sgds{<br>       jdbcurl = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-sgds-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>                   drivername = &quot;org.postgresql.Driver&quot;<br>                   username = &quot;appuser&quot;<br>                   password = &quot;AGZOgBuhOR9280CT&quot;<br>                   poolsize = 5<br>                   idletimeout = 120000<br>                   querytimeout=&quot;50000 millis&quot;<br>                   jdbcurlro = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-sgds-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>                   drivernamero = &quot;org.postgresql.Driver&quot;<br>                   usernamero = &quot;appuser&quot;<br>                   passwordro = &quot;AGZOgBuhOR9280CT&quot;<br>                   poolsizero = 5<br>                   idletimeoutro = 120000<br>                   querytimeoutro=&quot;50000 millis&quot;<br>                   jdbcurlwo = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-sgds-rds-cluster.cluster-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb_sgds?prepareThreshold=0&quot;<br>                   drivernamewo = &quot;org.postgresql.Driver&quot;<br>                   usernamewo = &quot;appuser&quot;<br>                   passwordwo = &quot;AGZOgBuhOR9280CT&quot;<br>                   poolsizewo = 5<br>                   idletimeoutwo = 120000<br>                   querytimeoutwo=&quot;50000 millis&quot;<br>                   databasename=&quot;ymdb_sgds&quot;<br>    }<br>        }<br>}</pre><pre>couchbaseconfig {<br>couchdbconfig = {<br>default {<br>nodes = [&quot;&quot;],<br>username = &quot;Administrator&quot;,<br>password = &quot;marriott123&quot;,<br>bucketpassword = &quot;&quot;<br>kvtimeout = &quot;10000000&quot;,<br>connectiontimeout = &quot;100000&quot;,<br>socketconnect = &quot;100000&quot;<br>}<br>}<br>}</pre><pre>brokerconfig {<br>kafkaconfig {<br>default {<br>bootstrapserver = &quot;kafka3-devtest1-aws-us-east-1.aws-use1.cloud.marriott.com:9093&quot;<br>kafkakeyserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;<br>kafkavalueserializer = &quot;org.apache.kafka.common.serialization.ByteArraySerializer&quot;<br>kafkasecurityprotocol = &quot;SSL&quot;<br>kafkassltruststorelocation = &quot;C:/Users/ajain304/Documents/EEH/eehDevTrustStore.jks&quot;<br>kafkasslkeystorelocation = &quot;C:/Users/ajain304/Documents/EEH/eeh.kafka.revenue-management.yield-management.hurdle-points.set.producer.dev.marriott.com.jks&quot;<br>kafkassltruststorepassword = &quot;123456&quot;<br>kafkasslkeystorepassword = &quot;DataInge$t!0n&quot;<br>kafkaidentificationalgorithm = &quot;&quot;<br>kafkatruststoretype = &quot;&quot;<br>}<br>}<br>}</pre><pre>bucketconfig {<br>bucket = {<br>default = &quot;s3://rms-oyty-test-emr/&quot;<br>}<br>}<br>schemaconfig {<br>schemamap {<br>SchemaName = &quot;ymtest&quot;<br>}<br>}<br>serviceconfig {<br>  servicemap = {<br>    ApplicationRunIdServiceUrl = &quot;&quot;&quot;https://appdata-app-rms-oytydev4.dev-r.aws-use1.cloud.marriott.com/rms/application/v1/properties/propCodeKey/runid&quot;&quot;&quot;<br>    ApplicationFeatureServiceUrl = &quot;&quot;&quot;https://appdata-app-rms-oytydev4.dev-r.aws-use1.cloud.marriott.com/rms/application/v1/properties/propCodeKey/featurestatus&quot;&quot;&quot;<br>    ORAnalyticsServiceUrl = &quot;&quot;&quot;https://refresh-fcst-calc-rms-ortst2.dev-r.aws-use1.cloud.marriott.com/ngicapi/1.0/forecast/refresh/json&quot;&quot;&quot;<br>    BenchMarkRatesServiceUrl = &quot;&quot;&quot;https://rateods-app-rms-oytydev4.dev-r.aws-use1.cloud.marriott.com/rms/rateods/v1/properties/propCodeKey/benchmarkrates&quot;&quot;&quot;<br>    HurdleRevenueServiceUrl = &quot;&quot;&quot;https://inventorycontrol-app-rms-oytydev4.dev-r.aws-use1.cloud.marriott.com/rms/inventorycontrol/v1/properties/propCodeKey/stays/hurdles&quot;&quot;&quot;<br>    RateForecastPredictServiceUrl = &quot;&quot;&quot;https://kd4w9irm11.execute-api.us-east-1.amazonaws.com/default/ReadS3FileLambda/rate_forecast_predictions.json&quot;&quot;&quot;<br>    OptimizationWindowServiceUrl = &quot;&quot;&quot;https://implementvmcl-rms-ortst2.dev-r.aws-use1.cloud.marriott.com/implement/window/propCodeKey/start&quot;&quot;&quot;<br>    CoreImplementsServiceUrl = &quot;&quot;&quot;https://kd4w9irm11.execute-api.us-east-1.amazonaws.com/default/ReadS3FileLambda/CoreImplimentService.json?propCode=propCodeKey&amp;runId=runIdKey&quot;&quot;&quot;<br>    KeyStoreName = &quot;RefreshImplementMGPDevKeystore.jks&quot;<br>    KeyStorePassword = &quot;h8sqt8Ws8UCzTest&quot;<br>    NewKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br>    NewKeyStorePassword = &quot;h8sqt8Ws8UCzTest&quot;<br> OptimizationWindowKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br> OptimizationWindowKeyStorePass=&quot;h8sqt8Ws8UCzTest&quot;<br> RateForecastPredictServiceKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br> RateForecastPredictServicePass=&quot;h8sqt8Ws8UCzTest&quot;<br> HurdleRevenueServiceKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br> HurdleRevenueServicePass=&quot;h8sqt8Ws8UCzTest&quot;<br> ORAnalyticsServiceKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br> ORAnalyticsServicePass=&quot;h8sqt8Ws8UCzTest&quot;<br> BenchMarkRatesServiceKeyStore=&quot;RefreshImplementMGPDevKeystore.jks&quot;<br> BenchMarkRatesServiceKeyPass=&quot;h8sqt8Ws8UCzTest&quot;</pre><pre>  }<br>}</pre><p>job.properties</p><pre>#---------------------------Enviroment----------------------------------#<br>nameNode=hdfs://ip-10-212-77-33.aws-use1.cloud.marriott.com:8020<br>jobTracker=ip-10-212-77-33.aws-use1.cloud.marriott.com:8032<br>queueName=default</pre><pre>#------------------------Application Path-------------------------------#<br>baseBucket1=rms-oyty-dev-emr<br>baseBucket=rms-oyty-dev-emr/oyc-test<br>FLOW_NAME=OYV2/DAILY_PROPERTY_BATCH<br>BASE_ORCH_PATH=s3://${baseBucket1}/OozieOrchestrationFlows<br>basePath=${BASE_ORCH_PATH}/${FLOW_NAME}<br>WORKFLOW_PATH=${basePath}/PSINVENTORY4_COMMONSTEP_RULE12.xml<br>JAVAAGENT=s3://rms-oyty-dev-emr/platform-team-tests/w3c/opentelemetry-javaagent-1.31.0.jar</pre><pre>#------------------------Python Configurations &amp; Arguments--------------------------------#<br>PREPROC_SCRIPT=PreProc.py<br>PREPROC_SCRIPT_PATH=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LATEST/${PREPROC_SCRIPT}<br></pre><pre>#-------------------------Oozie Configuration---------------------------------#<br>oozie.use.system.libpath=true<br>oozie.wf.validate.ForkJoin=false<br>oozie.wf.application.path=${WORKFLOW_PATH}<br>oozie.action.sharelib.for.spark=spark</pre><pre>#------------------------Scripts &amp; Configuration--------------------------------#<br>UTIL_FILE=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LATEST/utility.py<br>#UTIL_FILE=${basePath}/lib/utility.py<br>CRED_FILE=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LATEST/credentials.properties<br>#CRED_FILE=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LatestAnkit/credentials.properties<br>#JOB_CONFIG_FILE=${basePath}/lib/jobconfig.properties<br>JOB_CONFIG_FILE=${basePath}/lib/jobconfig_header.properties<br>CERT_FILE=${basePath}/lib/qa-keystore.cer<br>PRE_PROC_FILE=pre_proc_CFP.py<br>PRE_PROC_FILE_PATH=${basePath}/lib/${PRE_PROC_FILE}</pre><pre>POST_PROC_FILE=post_proc.py<br>POST_PROC_FILE_PATH=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LATEST/${POST_PROC_FILE}</pre><pre>#-------------------------------- POST-PROC SCRIPT ------------------------------#<br>POST_PROC_SCRIPT=${POST_PROC_FILE}<br>POST_PROC_SCRIPT_PATH=${POST_PROC_FILE_PATH}<br>POST_PROC_ARG1=-i<br>PSINVENTORY4_COMMONSTEP_RULE12_POST_PROC_ARG2=${PSINVENTORY4_COMMONSTEP_RULE12_INPUT_JSON}<br></pre><pre>#------------------------Job &amp; Flow Configuration--------------------------------#</pre><pre>#DATA_SNAPSHOT_FLAG=ON<br>#OYV2_FLAG={{OYV2}}<br>#GPO_FLAG={{GPO}}<br>#OY_FLAG={{OY}}<br>#RPO_FLAG={{RPO}}<br>#REVPLAN_FLAG={{REVPLAN}}<br>#UDEMAND_FLAG=ON<br>#CI_ENABLED={{CI_FLAG}}<br>#GF_ENABLED={{GF_FLAG}}<br>#OPERA_FLAG={{OPERA_FLAG}}<br>#SFA_FLAG={{SFA_FLAG}}<br>#MONTHLY_FLAG={{MONTHLY_RUN}}</pre><pre>#-------------------------INPUT JSON PATH---------------------------------#</pre><pre>PSINVENTORY4_COMMONSTEP_RULE12_INPUT_JSON=${PROP_CODE}/${RUN_ID}/GPO/PSINVENTORY4_COMMONSTEP_RULE_1_2/CONF/input.json<br></pre><pre>#--------------------------PRE-PROCESS SCRIPT----------------------------#<br>PRE_PROC_SCRIPT=${PRE_PROC_FILE}<br>PRE_PROC_SCRIPT_PATH=${PRE_PROC_FILE_PATH}</pre><pre>GROUP_PRE_PROC_SCRIPT=${GROUP_PRE_PROC_FILE}<br>GROUP_PRE_PROC_SCRIPT_PATH=${GROUP_PRE_PROC_FILE_PATH}</pre><pre>GROUP_PRE_PROC_FILE=group_pre_proc_summary.py<br>GROUP_PRE_PROC_FILE_PATH=${basePath}/lib/${GROUP_PRE_PROC_FILE}</pre><pre>PRE_PROC_ARG1=-p<br>PRE_PROC_ARG2=${PROP_CODE}<br>PRE_PROC_ARG3=-r<br>PRE_PROC_ARG4=${RUN_ID}<br>PRE_PROC_ARG5=-j</pre><pre>#---------------------------GROUP JOB ID-----------------------------#</pre><pre>PSINVENTORY4_COMMONSTEP_RULE12_PRE_PROC_ARG6=${PSINVENTORY4_COMMONSTEP_RULE12_JOB_ID}</pre><pre>PSINVENTORY4_COMMONSTEP_RULE12_JOB_ID=CI_SGDS_ETL_PSInventory4_CommonStep_Rule_1_2<br></pre><pre>#-------------GROUP JOB ID-------------#</pre><pre>PSINVENTORY4_COMMONSTEP_RULE12_PRE_PROC_ARG8=${PSINVENTORY4_COMMONSTEP_RULE12_INPUT_JSON}</pre><pre>PRE_PROC_ARG7=-f</pre><pre>PRE_PROC_ARG9=-a<br>PRE_PROC_ARG10=${ASOF_DATE}<br>PRE_PROC_ARG11=-d<br>PRE_PROC_ARG12=${DATA_DATE}</pre><pre>TRACE_ID={{TRACE_ID}}<br>SPAN_ID={{SPAN_ID}}<br>PRE_PROC_ARG99=-t<br>PRE_PROC_ARG100=${TRACE_ID}<br>PRE_PROC_ARG101=-s<br>PRE_PROC_ARG102=${SPAN_ID}<br> <br>TRACE_ID_VALUE={{TRACE_ID}}<br>SPAN_ID_VALUE={{SPAN_ID}}</pre><pre>OTEL_ARG1=-trace<br>OTEL_ARG2=${TRACE_ID_VALUE}<br>OTEL_ARG3=-span<br>OTEL_ARG4=${SPAN_ID_VALUE}<br>OTEL_ARG11=-t<br>OTEL_ARG33=-s<br>OTEL_FILE=s3://${baseBucket1}/OozieOrchestrationFlows/OYV2/FRAMEWORK/LATEST/telemetry_setup.py</pre><pre>#---- RUN_ID PROP_CODE DATES ----#<br>RUN_ID={{RUN_ID}}<br>PROP_CODE={{PROP_CODE}}<br>ASOF_DATE={{ASOF_DATE}}<br>DATA_DATE={{DATA_DATE}}<br>TRACE_ID={{TRACE_ID}}<br>SPAN_ID={{SPAN_ID}}</pre><pre>PRE_PROC_ARG99=-t<br>PRE_PROC_ARG100=${TRACE_ID}<br>PRE_PROC_ARG101=-s<br>PRE_PROC_ARG102=${SPAN_ID}<br>TRACE_ID_VALUE={{TRACE_ID}}<br>SPAN_ID_VALUE={{SPAN_ID}}</pre><pre>#------------------------Spark Configuration &amp; Arguments--------------------------------#<br>master=yarn<br>mode=cluster</pre><pre>CONFIG_FOLDER_PATH=${nameNode}/user/hadoop/oozie-config</pre><pre>CONNECTION_FILE=connection.conf<br>#CONNECTION_FILE_PATH=s3://rms-oyty-dev-emr/S3ToTableGenericJob/latest/${CONNECTION_FILE}<br>#CONNECTION_FILE_PATH=s3://rms-oyty-dev-emr/oyc-test/sharding/connection_updated.conf<br>CONNECTION_FILE_PATH=s3://rms-oyty-dev-emr/oyc-test/sharding/connection/connection.conf<br></pre><pre>PSINVENTORY4_COMMONSTEP_RULE12_JOBNAME=PSINVENTORY4_COMMONSTEP_RULE_1_2-${RUN_ID}-${PROP_CODE}-${TRACE_ID}-${SPAN_ID}<br>CI_SGDS_ETL_PSInventory4_CommonStep_Rule_1_2_JOBNAME=CI_SGDS_ETL_PSInventory4_CommonStep_Rule_1_2-${RUN_ID}-${PROP_CODE}-${TRACE_ID}-${SPAN_ID}<br>SPARK_ARG1=-c<br>SPARK_ARG2=${CONNECTION_FILE}<br>SPARK_ARG3=-j<br>SPARK_ARG5=-i<br></pre><pre>#--------------------------------PSINVENTORY4_COMMONSTEP_RULE12 SPARK ------------------------------#<br>PSINVENTORY4_COMMONSTEP_RULE12_JOBCONFIG=jobconfig.conf<br>PSINVENTORY4_COMMONSTEP_RULE12_JOBCONFIG_FILE_PATH=s3://${baseBucket1}/${PSINVENTORY4_COMMONSTEP_RULE12_JOB_ID}/latest/${PSINVENTORY4_COMMONSTEP_RULE12_JOBCONFIG}<br>PSINVENTORY4_COMMONSTEP_RULE12_CLASS_NAME=com.marriott.elevate.spark.ci.psinventory4.commonsteprule12.jobs.Main<br>PSINVENTORY4_COMMONSTEP_RULE12_PATH_TO_JAR=s3://${baseBucket1}/${PSINVENTORY4_COMMONSTEP_RULE12_JOB_ID}/latest/${PSINVENTORY4_COMMONSTEP_RULE12_JOB_ID}-1.0.6.jar<br>#PSINVENTORY4_COMMONSTEP_RULE12_PATH_TO_JAR={{PSINVENTORY4_COMMONSTEP_RULE12_PATH_TO_JAR}}<br>PSINVENTORY4_COMMONSTEP_RULE12_OPTS=--conf spark.executor.extraJavaOptions=-javaagent:opentelemetry-javaagent-1.31.0.jar -Dotel.java.global-autoconfigure.enabled=true -Dotel.logs.exporter=none -Dotel.propagators=tracecontext,baggage -Dotel.javaagent.logging=application -Dotel.metrics.exporter=none -Dotel.traces.exporter=otlp,logging -Dotel.javaagent.debug=true -Dotel.service.name=SparkService -Dotel.exporter.otlp.traces.endpoint=https://irn08782.live.dynatrace.com/api/v2/otlp/v1/traces -Dotel.exporter.otlp.traces.protocol=http/protobuf -Dotel.exporter.otlp.headers=Authorization=&quot;Api-Token dt0c01.2M2WED4M64UIRRZH75TW5QVQ.2NXTSUOT4TKV5UN2OESW7XNJLJC3D6ON7S5TFVD43OWLUUMFK4GPMRJOWZOMP7T6&quot; --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=8 --driver-memory 2G --executor-memory 5G  --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=5 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true --conf spark.shuffle.service.enabled=true<br>#--conf spark.driver.extraJavaOptions=-javaagent:opentelemetry-javaagent-1.31.0.jar  -Dotel.java.global-autoconfigure.enabled=true -Dotel.logs.exporter=none -Dotel.propagators=tracecontext,baggage -Dotel.javaagent.logging=application -Dotel.metrics.exporter=none -Dotel.traces.exporter=otlp,logging -Dotel.javaagent.debug=true -Dotel.service.name=SparkService -Dotel.exporter.otlp.traces.endpoint=https://irn08782.live.dynatrace.com/api/v2/otlp/v1/traces-Dotel.exporter.otlp.traces.protocol=http/protobuf -Dotel.exporter.otlp.headers=Authorization=&quot;Api-Token dt0c01.2M2WED4M64UIRRZH75TW5QVQ.2NXTSUOT4TKV5UN2OESW7XNJLJC3D6ON7S5TFVD43OWLUUMFK4GPMRJOWZOMP7T6&quot; --conf spark.executor.extraJavaOptions=-javaagent:opentelemetry-javaagent-1.31.0.jar -Dotel.java.global-autoconfigure.enabled=true -Dotel.logs.exporter=none -Dotel.propagators=tracecontext,baggage -Dotel.javaagent.logging=application -Dotel.metrics.exporter=none -Dotel.traces.exporter=otlp,logging -Dotel.javaagent.debug=true -Dotel.service.name=SparkService -Dotel.exporter.otlp.traces.endpoint=https://irn08782.live.dynatrace.com/api/v2/otlp/v1/traces-Dotel.exporter.otlp.traces.protocol=http/protobuf -Dotel.exporter.otlp.headers=Authorization=&quot;Api-Token dt0c01.2M2WED4M64UIRRZH75TW5QVQ.2NXTSUOT4TKV5UN2OESW7XNJLJC3D6ON7S5TFVD43OWLUUMFK4GPMRJOWZOMP7T6&quot; --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=8 --driver-memory 12G --executor-memory 10G  --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4  --conf spark.dynamicAllocation.enabled=true --conf spark.shuffle.service.enabled=true<br>#--conf spark.sql.shuffle.partitions=4 --driver-memory 1G --executor-memory 4G --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true<br># --conf spark.sql.shuffle.partitions=4 --driver-memory 1G --executor-memory 4G --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true --conf spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version=2 --conf spark.task.maxFailures=8 --conf spark.rpc.message.maxSize=1024 --conf spark.executor.extraJavaOptions=-XX:+UseG1GC --conf spark.network.timeout=240s --conf spark.shuffle.compress=true --conf spark.shuffle.spill.compress=true --conf spark.driver.extraJavaOptions=-DjobRunningMode=SparkBatchMode --conf spark.io.compression.codec=snappy --conf spark.rpc.askTimeout=600s<br>PSINVENTORY4_COMMONSTEP_RULE12_ARG4=${PSINVENTORY4_COMMONSTEP_RULE12_JOBCONFIG}<br>PSINVENTORY4_COMMONSTEP_RULE12_ARG6=s3://${baseBucket1}/${PSINVENTORY4_COMMONSTEP_RULE12_INPUT_JSON}</pre><pre>#------------------------------- FAILURE EMAIL -----------------------------------#<br>FAILURE_EMAIL_TO=DL-HDQBEElevateSupport@marriott.com<br>FAILURE_EMAIL_CC=DL-HDQBEElevateSupport@marriott.com<br>FAILURE_EMAIL_SUBJECT=${PSINVENTORY4_COMMONSTEP_RULE12_JOBNAME} Flow Failed</pre><p>SPB Bugs</p><pre>test env -- 12 jobs<br></pre><pre>similar story ---4 properties<br></pre><pre>issue tracker: troubleshooting fix</pre><pre>this is the sample command:<br> aws s3api list-objects-v2 --bucket rms-oyty-test-emr --prefix &quot;CHIMT/&quot; | jq -r &#39;.Contents[] | select(.Key | test(&quot;CHIMT/.*/REFRESH/DEM_UNC_STYDT_FCST_MODEL_OUT&quot;) and (contains(&quot;_SUCCESS&quot;) | not)) | .Key&#39; | sort -r | head -n 10</pre><pre> these are the paths on which we need to apply the above command individually</pre><pre> s3://&lt;bucket&gt;/&lt;propCode&gt;/&lt;run_id&gt;/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br> <br>s3://&lt;MGP_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30148646675.gzip<br> <br>s3://&lt;MGP_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT<br> <br>s3://&lt;Elevate_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br> <br>S3://&lt;Elevate_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/IMPLEMENT/unc_arvl_room_fcst</pre><pre> but for all these paths we dont have propcode like in sample command we have CHIMT, we dont have run id, we just have the path after run id as shown in the s3 path</pre><pre> modify the sample command indivially for all these s3 sample paths.<br></pre><pre> now suppose this is pathe */OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/* that we have to find in bucket rms-oyty-test-emr write a simple command for the same</pre><pre> ================================================1st===========================================================================================</pre><pre> TRAN_RATE_FCST_PRED<br></pre><pre> val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/30069786793/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet&quot;).printSchema()</pre><pre> aws s3api list-objects-v2 --bucket rms-oyty-test-emr --query &#39;Contents[?contains(Key, `OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/`)].Key&#39; --output text | head -n 10<br></pre><pre> ABETS/30057817528/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/<br>ABETS/30069786793/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>ABETS/30069827183/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  ABETS/30069827207/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069827208/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069827390/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  ABETS/30069827506/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069827804/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069827815/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>ABETS/30069847961/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  ABETS/30069849651/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069852410/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069853757/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  ABETS/30069855284/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069856947/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069858047/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        ABETS/30069859733/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30078941025/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30079101310/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30079248509/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30079406169/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30079577623/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30079735818/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30079889690/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30080033459/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30080196324/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30080346198/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30080496555/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30143126823/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30143387341/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30143628521/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30143901611/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30144221421/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30144480873/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30144756124/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30145016763/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30145292845/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30145552578/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30145836208/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30146142341/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30146435519/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30146704961/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet  BNAOX/30146966297/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30147259182/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet        BNAOX/30147568872/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br>BNAOX/30147805297/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet</pre><pre>root<br> |-- prop_code: string (nullable = true)<br> |-- dfs: long (nullable = true)<br> |-- los_tier: long (nullable = true)<br> |-- los_min: long (nullable = true)<br> |-- los_max: long (nullable = true)<br> |-- rc: long (nullable = true)<br> |-- stay_date: date (nullable = true)<br> |-- run_id: double (nullable = true)<br> |-- target_min357: double (nullable = true)<br> |-- target_min364: double (nullable = true)<br> |-- target_min371: double (nullable = true)<br> |-- target_min721: double (nullable = true)<br> |-- target_min728: double (nullable = true)<br> |-- target_min735: double (nullable = true)<br> |-- days_left: double (nullable = true)<br> |-- shop_rate: float (nullable = true)<br> |-- shop_date: date (nullable = true)<br> |-- fcst_rate: float (nullable = true)<br> |-- sys_fcst: float (nullable = true)<br> |-- net_rate: double (nullable = true)<br> |-- otb_adj_rate: double (nullable = true)<br> |-- adj_type_flag: string (nullable = true)<br> |-- __index_level_0__: long (nullable = true)<br></pre><pre>==================================================2nd==========================================================<br></pre><pre>UNC_ARVL_RATE_FCST</pre><pre> val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/BNAOX/30078941025/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30078941025.gzip&quot;).printSchema()<br></pre><pre>simillarly now suppose this is pathe */IMPLEMENT/unc_arvl_rate_fcst/* that we have to find in bucket rms-oyty-test-emr write a simple command for the same</pre><pre>s3://&lt;MGP_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30148646675.gzip<br></pre><pre>aws s3api list-objects-v2 --bucket rms-oyty-test-emr --query &#39;Contents[?contains(Key, `IMPLEMENT/unc_arvl_rate_fcst/`)].Key&#39; --output text | head -n 10</pre><pre>BNAOX/30078941025/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30078941025.gzip      BNAOX/30079101310/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079101310.gzip    BNAOX/30079248509/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079248509.gzip    BNAOX/30079406169/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079406169.gzip      BNAOX/30079577623/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079577623.gzip    BNAOX/30079735818/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079735818.gzip    BNAOX/30079889690/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30079889690.gzip      BNAOX/30080033459/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30080033459.gzip<br>BNAOX/30080196324/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30080196324.gzip      BNAOX/30080346198/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30080346198.gzip    BNAOX/30080496555/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30080496555.gzip<br>BNAOX/30143126823/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30143126823.gzip<br>BNAOX/30143387341/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30143387341.gzip      BNAOX/30143628521/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30143628521.gzip    BNAOX/30143901611/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30143901611.gzip    BNAOX/30144221421/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30144221421.gzip      BNAOX/30144480873/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30144480873.gzip    BNAOX/30144756124/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30144756124.gzip<br>BNAOX/30145016763/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30145016763.gzip      BNAOX/30145292845/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30145292845.gzip    BNAOX/30145552578/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30145552578.gzip    BNAOX/30145836208/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30145836208.gzip      BNAOX/30146142341/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30146142341.gzip    BNAOX/30146435519/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30146435519.gzip    BNAOX/30146704961/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30146704961.gzip<br>BNAOX/30146966297/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30146966297.gzip      BNAOX/30147259182/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30147259182.gzip    BNAOX/30147568872/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30147568872.gzip<br>BNAOX/30147805297/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30147805297.gzip      BNAOX/30148106635/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30148106635.gzip<br>BOSOF/30021582113/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30021582113.gzip      BOSOF/30021816521/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30021816521.gzip    BOSOF/30021887706/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30021887706.gzip<br>BOSOF/30021899639/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30021899639.gzip      BOSOF/30022124921/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022124921.gzip    BOSOF/30022212169/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022212169.gzip    BOSOF/30022274147/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022274147.gzip      BOSOF/30022438833/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022438833.gzip    BOSOF/30022506114/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022506114.gzip    BOSOF/30022604420/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022604420.gzip<br>BOSOF/30022710415/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022710415.gzip      BOSOF/30022782899/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022782899.gzip    BOSOF/30022791568/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022791568.gzip    BOSOF/30022791573/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022791573.gzip      BOSOF/30022853104/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30022853104.gzip    BOSOF/30023106670/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023106670.gzip    BOSOF/30023141450/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023141450.gzip      BOSOF/30023200570/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023200570.gzip    BOSOF/30023427645/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023427645.gzip    BOSOF/30023511627/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023511627.gzip      BOSOF/30023573191/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30023573191.gzip</pre><pre>root<br> |-- los: long (nullable = true)<br> |-- shop_date: date (nullable = true)<br> |-- net_rate: double (nullable = true)<br> |-- target_min735: double (nullable = true)<br> |-- target_min371: double (nullable = true)<br> |-- sys_fcst: float (nullable = true)<br> |-- shop_rate: float (nullable = true)<br> |-- arrival_date: timestamp (nullable = true)<br> |-- otb_adj_rate: double (nullable = true)<br> |-- days_left: integer (nullable = true)<br> |-- target_min728: double (nullable = true)<br> |-- prop_code: string (nullable = true)<br> |-- demand_forecast_segment_id: long (nullable = true)<br> |-- run_id: long (nullable = true)<br> |-- target_min721: double (nullable = true)<br> |-- adj_type_flag: string (nullable = true)<br> |-- target_min364: double (nullable = true)<br> |-- target_min357: double (nullable = true)<br> |-- room_pool_collection_id: long (nullable = true)<br> |-- rate: double (nullable = true)<br> |-- fcst_run_id: long (nullable = true)<br> |-- __index_level_0__: long (nullable = true)</pre><pre>=================================================3rd==========================================================</pre><pre>TRAN_ROOM_FCST_TRAIN_MODEL_OUT<br></pre><pre>simillarly now suppose this is pathe */OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/* that we have to find in bucket rms-oyty-test-emr write a simple command for the same</pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/20200626001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/ABETS_averages.parquet&quot;).printSchema()</pre><pre> aws s3api list-objects-v2 --bucket rms-oyty-test-emr --query &#39;Contents[?contains(Key, `OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/`)].Key&#39; --output text | head -n 10<br></pre><pre>ABETS/20200626001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/ABETS_averages.parquet<br>ABETS/20200918001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/ABETS_averages.parquet<br>ABETS/20201211001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/ABETS_averages.parquet<br>ABETS/20201212001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABETS/20210305001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABETS/20210514001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABETS/20210723001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABETS/20211002001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABETS/30069825380/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet       ABETS/30069825557/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/empty.parquet<br>ABQAP/20200626001/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT/ABQAP_averages.parquet</pre><pre>root<br> |-- month: string (nullable = true)<br> |-- dow: string (nullable = true)<br> |-- rpc: long (nullable = true)<br> |-- dfs: long (nullable = true)<br> |-- days_left: long (nullable = true)<br> |-- remaining_demand_avg: double (nullable = true)<br> |-- __index_level_0__: long (nullable = true)</pre><pre>====================================================================4th=======================================================================================<br></pre><pre>CNCL_FCST_MODEL_PCT</pre><pre>simillarly now suppose this is pathe */OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/* that we have to find in bucket rms-oyty-test-emr write a simple command for the same</pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/20210306001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet&quot;).printSchema()</pre><pre> aws s3api list-objects-v2 --bucket rms-oyty-test-emr --query &#39;Contents[?contains(Key, `OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/`)].Key&#39; --output text | head -n 10</pre><pre>ABETS/20210306001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABETS/20210514001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABETS/20210515001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABETS/20210516001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABETS/2882882/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABQCY/30000636554/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000637008/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABQCY/30000640719/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000643673/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000650622/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000657909/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000661572/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000668541/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000675882/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABQCY/30000684531/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000691290/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000700722/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000710933/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000719209/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000730760/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000739651/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABQCY/30000751270/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000761670/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000779909/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000792770/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000801988/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000813827/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000825039/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>ABQCY/30000845025/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet    ABQCY/30000858081/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000876127/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000893693/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet  ABQCY/30000906604/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br>AVLAK/20210514001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet</pre><pre>root<br> |-- PROP_CODE: string (nullable = true)<br> |-- STAY_DATE: timestamp (nullable = true)<br> |-- ASOF_DATE: timestamp (nullable = true)<br> |-- DAYS_OUT: long (nullable = true)<br> |-- ROOM_POOL_COLLECTION_ID: long (nullable = true)<br> |-- ON_TARGET_PERC: double (nullable = true)<br> |-- CANCEL_BUCKET_ID: string (nullable = true)<br> |-- CANCEL_PERC: double (nullable = true)<br> |-- CONSERVATIVE_PERC: double (nullable = true)<br> |-- __index_level_0__: long (nullable = true)<br></pre><pre>====================================================5th========================================================================================</pre><pre>UNC_ARVL_ROOM_FCST<br></pre><pre>simillarly now suppose this is pathe */IMPLEMENT/unc_arvl_room_fcst/* that we have to find in bucket rms-oyty-test-emr write a simple command for the same</pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/BNAOX/30078941025/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30078941025.gzip&quot;).printSchema()</pre><pre> aws s3api list-objects-v2 --bucket rms-oyty-test-emr --query &#39;Contents[?contains(Key, `IMPLEMENT/unc_arvl_room_fcst/`)].Key&#39; --output text | head -n 10</pre><pre> BNAOX/30078941025/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30078941025.gzip      BNAOX/30079101310/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079101310.gzip    BNAOX/30079248509/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079248509.gzip    BNAOX/30079406169/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079406169.gzip      BNAOX/30079577623/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079577623.gzip    BNAOX/30079735818/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079735818.gzip    BNAOX/30079889690/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30079889690.gzip      BNAOX/30080033459/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30080033459.gzip<br>BNAOX/30080196324/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30080196324.gzip      BNAOX/30080346198/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30080346198.gzip    BNAOX/30080496555/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30080496555.gzip<br>BNAOX/30143126823/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30143126823.gzip<br>BNAOX/30143387341/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30143387341.gzip      BNAOX/30143628521/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30143628521.gzip    BNAOX/30143901611/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30143901611.gzip    BNAOX/30144221421/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30144221421.gzip      BNAOX/30144480873/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30144480873.gzip    BNAOX/30144756124/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30144756124.gzip<br>BNAOX/30145016763/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30145016763.gzip      BNAOX/30145292845/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30145292845.gzip    BNAOX/30145552578/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30145552578.gzip    BNAOX/30145836208/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30145836208.gzip      BNAOX/30146142341/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30146142341.gzip    BNAOX/30146435519/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30146435519.gzip    BNAOX/30146704961/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30146704961.gzip<br>BNAOX/30146966297/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30146966297.gzip      BNAOX/30147259182/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30147259182.gzip    BNAOX/30147568872/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30147568872.gzip<br>BNAOX/30147805297/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30147805297.gzip      BNAOX/30148106635/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30148106635.gzip<br>BOSOF/30021582113/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30021582113.gzip      BOSOF/30021816521/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30021816521.gzip    BOSOF/30021887706/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30021887706.gzip<br>BOSOF/30021899639/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30021899639.gzip      BOSOF/30022124921/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022124921.gzip    BOSOF/30022212169/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022212169.gzip    BOSOF/30022274147/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022274147.gzip      BOSOF/30022438833/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022438833.gzip    BOSOF/30022506114/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022506114.gzip    BOSOF/30022604420/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022604420.gzip<br>BOSOF/30022710415/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022710415.gzip      BOSOF/30022782899/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022782899.gzip    BOSOF/30022791568/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022791568.gzip    BOSOF/30022791573/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022791573.gzip      BOSOF/30022853104/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30022853104.gzip    BOSOF/30023106670/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023106670.gzip    BOSOF/30023141450/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023141450.gzip      BOSOF/30023200570/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023200570.gzip    BOSOF/30023427645/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023427645.gzip    BOSOF/30023511627/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023511627.gzip      BOSOF/30023573191/IMPLEMENT/unc_arvl_room_fcst/unc_arvl_room_fcst_30023573191.gzip<br></pre><pre>root<br> |-- room_pool_collection_id: long (nullable = true)<br> |-- demand_forecast_segment_id: long (nullable = true)<br> |-- los: long (nullable = true)<br> |-- uncon_rem_forecast: double (nullable = true)<br> |-- arrival_date: timestamp (nullable = true)<br> |-- last_upd_by: string (nullable = true)<br> |-- last_upd_stamp: timestamp (nullable = true)</pre><pre>===============================================<br></pre><pre>this is the scehma in the parquet file <br>root<br> |-- PROP_CODE: string (nullable = true)<br> |-- STAY_DATE: timestamp (nullable = true)<br> |-- ASOF_DATE: timestamp (nullable = true)<br> |-- DAYS_OUT: long (nullable = true)<br> |-- ROOM_POOL_COLLECTION_ID: long (nullable = true)<br> |-- ON_TARGET_PERC: double (nullable = true)<br> |-- CANCEL_BUCKET_ID: string (nullable = true)<br> |-- CANCEL_PERC: double (nullable = true)<br> |-- CONSERVATIVE_PERC: double (nullable = true)<br> |-- __index_level_0__: long (nullable = true)</pre><pre> which we have received via the command <br>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/20210306001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet&quot;).printSchema()</pre><pre>now i want to know ahich job is using this and what is the significance of this column  |-- __index_level_0__: long (nullable = true)? how can i know  ,..what commands i can use for more info like knowing the jobs which are using this </pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/30069786793/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet&quot;)<br>df.show(10)<br></pre><pre>+---------+---+--------+-------+-------+---+----------+---------------+-------------+-------------+-------------+-------------+-------------+-------------+---------+---------+----------+----------+----------+--------+------------+-------------+-----------------+<br>|prop_code|dfs|los_tier|los_min|los_max| rc| stay_date|         run_id|target_min357|target_min364|target_min371|target_min721|target_min728|target_min735|days_left|shop_rate| shop_date| fcst_rate|  sys_fcst|net_rate|otb_adj_rate|adj_type_flag|__index_level_0__|<br>+---------+---+--------+-------+-------+---+----------+---------------+-------------+-------------+-------------+-------------+-------------+-------------+---------+---------+----------+----------+----------+--------+------------+-------------+-----------------+<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-15|3.0069786793E10|         null|         null|         null|         null|         null|         null|     12.0|     null|2024-05-24|  77.11141|  77.11141|    null|        null|           NA|                0|<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-19|3.0069786793E10|         null|         null|         null|         null|         null|         null|     16.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                1|<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-21|3.0069786793E10|         null|         null|         null|         null|         null|         null|     18.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                2|<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-25|3.0069786793E10|         null|         null|         null|         null|         null|         null|     22.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                3|<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-26|3.0069786793E10|         null|         null|         null|         null|         null|         null|     23.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                4|<br>|    ABETS|  2|       3|      6|     12|  2|2021-04-28|3.0069786793E10|         null|         null|         null|         null|         null|         null|     25.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                5|<br>|    ABETS|  2|       3|      6|     12|  2|2021-05-04|3.0069786793E10|         null|         null|         null|         null|         null|         null|     31.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                6|<br>|    ABETS|  2|       3|      6|     12|  2|2021-05-08|3.0069786793E10|         null|         null|         null|         null|         null|         null|     35.0|     null|2024-05-24|122.274086|122.274086|    null|        null|           NA|                7|<br>|    ABETS|  2|       3|      6|     12|  2|2021-05-14|3.0069786793E10|         null|         null|         null|         null|         null|         null|     41.0|     null|2024-05-24| 118.70763| 118.70763|    null|        null|           NA|                8|<br>|    ABETS|  2|       3|      6|     12|  2|2021-05-22|3.0069786793E10|         null|         null|         null|         null|         null|         null|     49.0|     null|2024-05-24|  128.2727|  128.2727|    null|        null|           NA|                9|<br>+---------+---+--------+-------+-------+---+----------+---------------+-------------+-------------+-------------+-------------+-------------+-------------+---------+---------+----------+----------+----------+--------+------------+-------------+-----------------+<br></pre><pre>================================================================================<br></pre><pre>I access to the logs of various jobs running on your cluster, you can search for the path s3://rms-oyty-test-emr/ABETS/30069786793/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet to see which jobs are reading this file.<br>give me a spark command to find which job is using this command<br></pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/ABETS/30069786793/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet&quot;)</pre><pre>===============================================================1st job====================================================================================================<br></pre><pre>   latestRateFCSTPredQuery=&quot;&quot;&quot;(select sd_sk<br>              ,sd_entity<br>              ,sd_domain<br>              ,sd_feature_id<br>              ,sd_dataset_id<br>              ,sd_run_id<br>              ,sd_asof_date<br>              ,sd_run_date<br>              ,sd_data_date_from<br>              ,sd_data_date_to<br>              ,sd_applicable_date_from<br>              ,sd_applicable_date_to<br>              ,sd_valid_from<br>              ,sd_valid_to<br>              ,sd_warning_date<br>              ,sd_dataset_path<br>              ,sd_source_dataset_sk<br>              ,sd_active<br>              ,sd_notes<br>              ,sd_last_upd_by<br>              ,sd_last_upd_ts<br>           from ymtest.rms_shared_dataset<br>           where sd_entity = &#39;propCodeKey&#39;<br>           and sd_dataset_id = &#39;TRAN_RATE_FCST_PRED&#39;<br>           and sd_feature_id = &#39;OYV2&#39;<br>           and sd_domain = &#39;PROPERTY&#39;<br>           and exists(SELECT 1 FROM ymtest.rms_data_status<br>              where ds_process_action = &#39;IMPLEMENT&#39;<br>              and ds_run_id &gt;=<br>                (select DS_RUN_ID<br>                from ymtest.rms_data_status<br>                where ds_process_action = &#39;IMPLEMENT&#39;<br>                and ds_prop_code = &#39;propCodeKey&#39;<br>                and ds_data_date &gt;= &#39;childStartDateKey&#39;::date - &#39;15 days&#39;::interval<br>                and ds_applicable_date_from &lt;= &#39;childStartDateKey&#39;::date<br>                and &#39;childStartDateKey&#39;::date + horizonKey * &#39;1 day&#39;::interval &lt;= ds_applicable_date_to + &#39;15 days&#39;::interval<br>                ORDER BY DS_RUN_ID DESC LIMIT 1)<br>              and ds_prop_code = sd_entity<br>              and ds_run_id = sd_run_id<br>             )<br>           order by sd_asof_date,sd_run_id)as latest_rate_fcst_pred&quot;&quot;&quot;<br></pre><pre>TRAN_RATE_FCST_PRED s3://&lt;bucket&gt;/&lt;propCode&gt;/&lt;run_id&gt;/OYV2/TRAN_RATE_FCST_PRED/MODEL_OUT/rate_forecast_predictions.parquet<br> <br>UNC_ARVL_RATE_FCST s3://&lt;MGP_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/IMPLEMENT/unc_arvl_rate_fcst/unc_arvl_rate_fcst_30148646675.gzip<br> <br>TRAN_ROOM_FCST_TRAIN_MODEL_OUT s3://&lt;MGP_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/OYV2/ROOM_FRCST_REGRS_WITH_LAGS/ROOM_FCST_TRAIN_MODEL_OUT<br> <br>CNCL_FCST_MODEL_PCT s3://&lt;Elevate_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br> <br>UNC_ARVL_ROOM_FCST S3://&lt;Elevate_Bucket&gt;/&lt;PROP_CODE&gt;/&lt;RUN_ID&gt;/IMPLEMENT/unc_arvl_room_fcst<br></pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/20201211001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet&quot;)</pre><pre>df.show(10)</pre><pre>application_1727252683508_22098</pre><pre>yarn logs -applicationId application_1727252683508_22098 | grep -C 20 &quot;Error&quot;</pre><pre>   +- Relation [PROP_CODE#0,STAY_DATE#1,ASOF_DATE#2,DAYS_OUT#3L,ROOM_POOL_COLLECTION_ID#4L,ON_TARGET_PERC#5,CANCEL_BUCKET_ID#6,CANCEL_PERC#7,CONSERVATIVE_PERC#8,__index_level_0__#9L] parquet\n&quot;,&quot;logger_name&quot;:&quot;com.marriott.elevate.s3totable.jobs.S3ToTableLoadJob&quot;,&quot;thread_name&quot;:&quot;Driver&quot;,&quot;level&quot;:&quot;ERROR&quot;,&quot;level_value&quot;:40000,&quot;stack_trace&quot;:&quot;org.apache.spark.sql.</pre><pre>   AnalysisException: Column &#39;ROOM_COLLECTION_ID&#39; does not exist. </pre><pre>   Did you mean one of the following? [ROOM_POOL_COLLECTION_ID, CANCEL_BUCKET_ID, CFP_RUN_ID, PROP_CODE, RUN_ID, ON_TARGET_PERC, ASOF_DATE, CFP_CANCEL_BUCKET_ID, CFP_LAST_UPD_BY, CFP_LAST_UPD_TS, CFP_PROP_CODE, DAYS_OUT, CANCEL_PERC, CFP_ASOF_DATE, CFP_DAYS_OUT, CFP_ON_TARGET_PERC, CONSERVATIVE_PERC, STAY_DATE, __index_level_0__]; </pre><pre>   line 1 pos 0;\n&#39;Project [PROP_CODE#0, STAY_DATE#1, ASOF_DATE#2, DAYS_OUT#3L, ROOM_POOL_COLLECTION_ID#4L, ON_TARGET_PERC#5, CANCEL_BUCKET_ID#6, CANCEL_PERC#7, CONSERVATIVE_PERC#8, __index_level_0__#9L, RUN_ID#20, CFP_ASOF_DATE#90, CFP_LAST_UPD_TS#103, CFP_LAST_UPD_BY#117, CFP_DAYS_OUT#132L, CFP_CANCEL_BUCKET_ID#148, CFP_RUN_ID#165L, CFP_ON_TARGET_PERC#183, CFP_PROP_CODE#202, &#39;ROOM_COLLECTION_ID AS cfp_room_collection_id#222]\n+- Project [PROP_CODE#0, STAY_DATE#1, ASOF_DATE#2, DAYS_OUT#3L, ROOM_POOL_COLLECTION_ID#4L, ON_TARGET_PERC#5, CANCEL_BUCKET_ID#6, CANCEL_PERC#7, CONSERVATIVE_PERC#8, __index_level_0__#9L, RUN_ID#20, CFP_ASOF_DATE#90, CFP_LAST_UPD_TS#103, CFP_LAST_UPD_BY#117, CFP_DAYS_OUT#132L, CFP_CANCEL_BUCKET_ID#148, CFP_RUN_ID#165L, CFP_ON_TARGET_PERC#183, PROP_CODE#0 AS CFP_PROP_CODE#202]\n   +- Project [PROP_CODE#0, STAY_DATE#1, ASOF_DATE#2, DAYS_OUT#3L, ROOM_POOL_COLLECTION_ID#4L, ON_TARGET_PERC#5, CANCEL_BUCKET_ID#6, CANCEL_PERC#7, CONSERVATIVE_PERC#8, __index_level_0__#9L, RUN_ID#20, CFP_ASOF_DATE#90, CFP_LAST_UPD_TS#103, CFP_LAST_UPD_BY#117, CFP_DAYS_OUT#132L, CFP_CANCEL_BUCKET_ID#148, CFP_RUN_ID#165L, cast(ON_TARGET_PERC#5 as double) AS CFP_ON_TARGET_PERC#183]\n      +- Project [PROP_CODE#0, STAY_DATE#1, ASOF_DATE#2, DAYS_OUT#3L, ROOM_POOL_COLLECTION_ID#4L, ON_TARGET_PERC#5, CANCEL_BUCKET_ID#6, CANCEL_PERC#7, CONSERVATIVE_PERC#8, __index_level_0__#9L, RUN_ID#20, CF<br></pre><pre>path:<br></pre><pre>in logs<br></pre><pre>   SPICY/30147742339/OYV2/CANCELLATION_FORECAST_PERCENTAGE/CONF/input.json<br></pre><pre>in input.json</pre><pre>   SPICY/20201211001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br></pre><pre>   val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/20201211001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet&quot;)</pre><pre>   df.show(10)<br></pre><pre>   +---------+-------------------+-------------------+--------+-----------------------+--------------+----------------+------------------+------------------+-----------------+<br>|PROP_CODE|          STAY_DATE|          ASOF_DATE|DAYS_OUT|ROOM_POOL_COLLECTION_ID|ON_TARGET_PERC|CANCEL_BUCKET_ID|       CANCEL_PERC| CONSERVATIVE_PERC|__index_level_0__|<br>+---------+-------------------+-------------------+--------+-----------------------+--------------+----------------+------------------+------------------+-----------------+<br>|    SPICY|2024-07-11 00:00:00|2024-07-11 00:00:00|       0|                      1|           0.2|               C|               0.0|               0.0|                0|<br>|    SPICY|2024-07-11 00:00:00|2024-07-11 00:00:00|       0|                      2|           0.2|               C|               0.0|               0.0|                1|<br>|    SPICY|2024-07-12 00:00:00|2024-07-11 00:00:00|       1|                      1|           0.2|               C|1.4492753623188406|               0.0|                2|<br>|    SPICY|2024-07-12 00:00:00|2024-07-11 00:00:00|       1|                      2|           0.2|               C|1.4492753623188406|               0.0|                3|<br>|    SPICY|2024-07-13 00:00:00|2024-07-11 00:00:00|       2|                      1|           0.2|               C| 7.042253521126761| 2.816901408450704|                4|<br>|    SPICY|2024-07-13 00:00:00|2024-07-11 00:00:00|       2|                      2|           0.2|               C| 7.042253521126761| 2.816901408450704|                5|<br>|    SPICY|2024-07-14 00:00:00|2024-07-11 00:00:00|       3|                      1|           0.2|               C| 4.761904761904762| 2.380952380952381|                6|<br>|    SPICY|2024-07-14 00:00:00|2024-07-11 00:00:00|       3|                      2|           0.2|               C| 4.761904761904762| 2.380952380952381|                7|<br>|    SPICY|2024-07-15 00:00:00|2024-07-11 00:00:00|       4|                      1|           0.2|               C| 6.666666666666667|3.3333333333333335|                8|<br>|    SPICY|2024-07-15 00:00:00|2024-07-11 00:00:00|       4|                      2|           0.2|               C| 6.666666666666667|3.3333333333333335|                9|<br>+---------+-------------------+-------------------+--------+-----------------------+--------------+----------------+------------------+------------------+-----------------+</pre><pre>using the mockfile for bosog which is working </pre><pre>s3://rms-oyty-test-emr/BOSOF/20201211001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet</pre><pre>DOWNLOADED IT<br></pre><pre>GOING TO THIS PATH</pre><pre>s3://rms-oyty-test-emr/SPICY/20201211001/OYV2/CANCELLATION_FORECAST/MODEL_OUTPUT/CNCL_FCST_MODEL_PCT/cancel_pct_stg.parquet<br></pre><pre>now getting the error :</pre><pre>{&quot;@timestamp&quot;:&quot;2024-09-30T06:48:42.18Z&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Process Error: org.apache.spark.sql.AnalysisException: Column &#39;rc&#39; does not exist. Did you mean one of the following? [rpc, rooms, dfsId, propCode, daysLeft, stayDate, roomsRemaining, paidGrossCancels, remainingDemand, additionalDemand, paidNetOnTheBooks, paidGrossOnTheBooks, additionalDemandRemaining, cummulativeAdditionalDemand];\n&#39;Project [propCode#1603, &#39;rc, dfsId#1605, stayDate#1606, daysLeft#1607, rooms#1608, additionalDemand#1609, roomsRemaining#1610, additionalDemandRemaining#1611, remainingDemand#1612, paidNetOnTheBooks#1613, paidGrossCancels#1614, cummulativeAdditionalDemand#1615</pre><pre> s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/input.json<br></pre><pre> got two input paths<br>first one for the current eun<br>2nd one is of previous run</pre><pre>which we corrected by taking all the parquet files uploaded for the NYCIF flow</pre><pre> &quot;inputdatapath&quot;: [<br>  {<br>   &quot;sdsk&quot;: 1100496134,<br>   &quot;runid&quot;: 30147758923,<br>   &quot;datadateto&quot;: &quot;2024-07-26&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-27&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-28&quot;,<br>   &quot;validto&quot;: &quot;2024-07-27 06:43:33.879+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-26 06:43:33.879+00&quot;,<br>   &quot;datasetid&quot;: &quot;STAYDT_DFS_RC_AGG&quot;<br>  },<br>  {<br>   &quot;sdsk&quot;: 1100482303,<br>   &quot;runid&quot;: 30148662105,<br>   &quot;datadateto&quot;: &quot;2024-07-25&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-25&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-27&quot;,<br>   &quot;validto&quot;: &quot;2024-07-26 08:12:46.674+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-25 08:12:46.674+00&quot;,<br>   &quot;datasetid&quot;: &quot;TRAN_REM_DMD_BY_DL&quot;<br>  }<br> ],<br></pre><pre>for first path this is the file :s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/</pre><pre>s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/part-00000-a84b9a9b-f23d-446d-97a3-b9cc2aba2043-c000.snappy.parquet<br></pre><pre> val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/part-00000-a84b9a9b-f23d-446d-97a3-b9cc2aba2043-c000.snappy.parquet&quot;)<br></pre><pre>df.show(10)<br></pre><pre>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>|SnapShotDate|rba_prop_code|StayDate|rba_bkng_date|rcd_room_collection_id|rpsm_dem_fcst_seg_id|lostierdata|additional_dmd|rba_currency_code|rba_stay_rms|rba_pay_rms|rba_comp_rms|rba_cncl_pay_rms|rba_cncl_pay_rev|rba_cncl_stay_rms|rba_cncl_stay_rev|rba_stay_gross_rev|rba_stay_net_rev|rba_pay_gross_rev|rba_pay_net_rev|rba_total_pay_gross_rev|rba_total_pay_net_rev|rba_total_stay_gross_rev|rba_total_stay_net_rev|net_rms_for_rate|net_rms_for_demand|<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+</pre><pre>2nd dataset path:</pre><pre>val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/part-00000-3e79c85c-91b7-4a47-86e4-581edf3de58c-c000.snappy.parquet&quot;)<br></pre><pre>AS THERE ARE MULTIPLE PARQUET FILE , WE CAN CHECK THEM ALL USING</pre><pre>scala&gt; val df = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/*&quot;)<br>df: org.apache.spark.sql.DataFrame = [propCode: string, rpc: int ... 12 more fields]</pre><pre>scala&gt; df.show(10)<br></pre><pre>scala&gt; df.count<br>res4: Long = 38186544</pre><pre>scala&gt; val df1 = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/part-00000-3e79c85c-91b7-4a47-86e4-581edf3de58c-c000.snappy.parquet&quot;)<br>df1: org.apache.spark.sql.DataFrame = [propCode: string, rpc: int ... 12 more fields]</pre><pre>scala&gt; df1.count<br>res5: Long = 13521901</pre><pre>its failing for the job ADDITIONAL_DEMAND_LOADS_EVENTS_DATA......connection issue </pre><pre>Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ip-10-215-65-99.aws-use1.cloud.marriott.com/10.215.65.99:37363<br>Caused by: java.net.ConnectException: Connection refused<br>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)<br>        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)<br>        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)<br></pre><pre>        after that </pre><pre>        2024-09-30 09:21:28,596 [main] ERROR org.apache.spark.deploy.yarn.Client  - Application diagnostics message: User class threw exception: java.io.FileNotFoundException: No such file or directory &#39;s3://rms-oyty-test-emr/SPICY/30147760310/UDEMAND/ADDITIONAL_DEMAND_LOADS_EVENTS_DATA/CONF/input.json&#39;</pre><pre>chnaged the job config of the job<br></pre><pre>Rerun issue of duplicate:</pre><pre>now getting the error :</pre><pre> Job aborted due to stage failure: Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in stage 1.0 (TID 4) (ip-10-215-65-91.aws-use1.cloud.marriott.com executor 1): java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.TRAN_ROOM_SUPPLY (\&quot;trs_run_id\&quot;,\&quot;trs_prop_code\&quot;,\&quot;trs_stay_date\&quot;,\&quot;trs_inventory_type\&quot;,\&quot;trs_room_rms_supply\&quot;,\&quot;trs_last_upd_by\&quot;,\&quot;trs_last_upd_ts\&quot;) VALUES (30147761159,&#39;SPICY&#39;,&#39;2024-09-23 +00&#39;,&#39;GENR&#39;,45,&#39;PoprTransAuth&#39;,&#39;2024-09-30 11:39:41.301+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_trs\&quot;\n  Detail: Key (trs_prop_code, trs_run_id, trs_stay_date, trs_inventory_type)=(SPICY, 30147761159, 2024-09-23, GENR) already exists.  Call getNextException to see other errors in the batch.\n\tat org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)\n\tat org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2134)\n\tat org.postgresql.core.v3.QueryExecutorImpl.flushIfDeadlockRisk(QueryExecutorImpl.java:1491)\n\tat org.postgresql.core.v3.QueryExecutorImpl.sendQuery(QueryExecutorImpl.java:1516)\n\tat org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:560)\n\tat org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)\n\tat org.postgresql.jdbc.PgS</pre><pre> select * from ymtest.TRAN_ROOM_SUPPLY where trs_prop_code = &#39;SPICY&#39; , trs_run_id = &#39;30147761159&#39;, trs_stay_date = &#39;2024-09-23&#39;, trs_inventory_type = &#39;GENR&#39;</pre><pre>now the error :runid folder is not getting created</pre><pre>grep exception</pre><pre>CustomLoggerInfo(30147762712,SPICY,Revenue Management,OverrideMenuTargetLoad,OverrideMenuTargetLoad,S3: s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/,DataFrame,,,)<br>2024-09-30 13:50:00,135 [Driver] INFO  com.marriott.elevate.overridemenuoutputjob.jobs.OverrideMenuOutputJob$$anon$2  - Failed to read/write data due to: Path does not exist: s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT - Retries left: 3.<br>2024-09-30 13:50:00,136 [Driver] INFO  com.marriott.elevate.overridemenuoutputjob.jobs.OverrideMenuOutputJob$$anon$2  - Start reading from S3.<br>CustomLoggerInfo(30147762712,SPICY,Revenue Management,OverrideMenuTargetLoad,OverrideMenuTargetLoad,S3: s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/,DataFrame,,,)<br>2024-09-30 13:50:00,235 [Driver] INFO  com.marriott.elevate.overridemenuoutputjob.jobs.OverrideMenuOutputJob$$anon$2  - Failed to read/write data due to: Path does not exist: s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT - Retries left: 2.<br>2024-09-30 13:50:00,235 [Driver] INFO  com.marriott.elevate.overridemenuoutputjob.jobs.OverrideMenuOutputJob$$anon$2  - Start reading from S3.</pre><pre>grep error<br></pre><pre>After Transformation of sparkInput:Map(propcode -&gt; SPICY, asofdate -&gt; 2024-07-26, spanid -&gt; 6d57b272ae647dbb, entity -&gt; SPICY, domain -&gt; PROPERTY, inputdatapath -&gt; List(Map(sdsk -&gt; 15956187, runid -&gt; 30063177330, datadateto -&gt; 2023-04-10, warningdate -&gt; 2023-04-12, asofdate -&gt; 2023-04-11, datadatefrom -&gt; 2019-04-12, validto -&gt; 2023-04-12 19:03:44.165+00, datasetpath -&gt; s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/, validfrom -&gt; 2023-04-11 19:03:44.165+00, datasetid -&gt; OVERRIDE_MENU_MODEL_OUT)), datadate -&gt; 2024-07-25, outputjsonpath -&gt; SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/RMS/, runid -&gt; 30147762712, jobid -&gt; OverrideMenuTargetLoad, currentdate -&gt; 2024-07-26, traceid -&gt; dc07e68ba3a31f537ad83adab7600f6f, outputdatapath -&gt; SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/)End<br>2024-09-30 13:49:52,338 [Driver] ERROR com.marriott.elevate.spark.framework.traits.ProcessHook  -<br></pre><pre> No reference files found in File Catalog for Job Id OverrideMenuTargetLoad</pre><pre>(TraceId: dc07e68ba3a31f537ad83adab7600f6f,397d55ff2e53c689 End)<br>(Traceconfig TraceId: dc07e68ba3a31f537ad83adab7600f6f,6d57b272ae647dbb End)<br>After Transformation of sparkInput:Map(propcode -&gt; SPICY, asofdate -&gt; 2024-07-26, spanid -&gt; 6d57b272ae647dbb, entity -&gt; SPICY, domain -&gt; PROPERTY, inputdatapath -&gt; List(Map(sdsk -&gt; 15956187, runid -&gt; 30063177330, datadateto -&gt; 2023-04-10, warningdate -&gt; 2023-04-12, asofdate -&gt; 2023-04-11, datadatefrom -&gt; 2019-04-12, validto -&gt; 2023-04-12 19:03:44.165+00, datasetpath -&gt; s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/, validfrom -&gt; 2023-04-11 19:03:44.165+00, datasetid -&gt; OVERRIDE_MENU_MODEL_OUT)), datadate -&gt; 2024-07-25, outputjsonpath -&gt; SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/RMS/, runid -&gt; 30147762712, jobid -&gt; OverrideMenuTargetLoad, currentdate -&gt; 2024-07-26, traceid -&gt; dc07e68ba3a31f537ad83adab7600f6f, outputdatapath -&gt; SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/)End<br>2024-09-30 13:49:52,428 [Driver] INFO  com.marriott.elevate.spark.framework.traits.ProcessHook  - jsonDataWrite= {&quot;propcode&quot;:&quot;SPICY&quot;,&quot;asofdate&quot;:&quot;2024-07-26&quot;,&quot;spanid&quot;:&quot;6d57b272ae647dbb&quot;,&quot;entity&quot;:&quot;SPICY&quot;,&quot;domain&quot;:&quot;PROPERTY&quot;,&quot;inputdatapath&quot;:[{&quot;sdsk&quot;:15956187,&quot;runid&quot;:30063177330,&quot;datadateto&quot;:&quot;2023-04-10&quot;,&quot;warningdate&quot;:&quot;2023-04-12&quot;,&quot;asofdate&quot;:&quot;2023-04-11&quot;,&quot;datadatefrom&quot;:&quot;2019-04-12&quot;,&quot;validto&quot;:&quot;2023-04-12 19:03:44.165+00&quot;,&quot;datasetpath&quot;:&quot;s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/&quot;,&quot;validfrom&quot;:&quot;2023-04-11 19:03:44.165+00&quot;,&quot;datasetid&quot;:&quot;OVERRIDE_MENU_MODEL_OUT&quot;}],&quot;datadate&quot;:&quot;2024-07-25&quot;,&quot;outputjsonpath&quot;:&quot;SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/RMS/&quot;,&quot;runid&quot;:30147762712,&quot;jobid&quot;:&quot;OverrideMenuTargetLoad&quot;,&quot;currentdate&quot;:&quot;2024-07-26&quot;,&quot;traceid&quot;:&quot;dc07e68ba3a31f537ad83adab7600f6f&quot;,&quot;outputdatapath&quot;:&quot;SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/&quot;}<br>2024-09-30 13:49:52,428 [Driver] INFO  com.marriott.elevate.spark.framework.traits.ProcessHook  - bucket and key to write data are bucket : s3a://rms-oyty-test-emr/ and key is Some(SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/input.json)<br>2024-09-30 13:49:52,429 [Driver] INFO  com.marriott.elevate.spark.framework.traits.ProcessHook  - objekey = SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/input.json</pre><pre>{&quot;@timestamp&quot;:&quot;2024-09-30T13:49:35.219Z&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Error ::: OverrideMenuTargetLoad Job Path does not exist: <br></pre><pre>s3://rms-oyty-test-emr/SPICY/30147762712/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/RMS&quot;,<br></pre><pre>&quot;logger_name&quot;:&quot;com.marriott.elevate.overridemenuoutputjob.jobs.Main$&quot;,&quot;thread_name&quot;:&quot;Driver&quot;,&quot;level&quot;:&quot;ERROR&quot;,&quot;level_value&quot;:40000,&quot;app&quot;:&quot;OverrideMenuTargetLoad&quot;,&quot;trace-id&quot;:&quot;traceId&quot;,&quot;project&quot;:&quot;Revenue Management&quot;,&quot;customData&quot;:&quot;{\&quot;source\&quot;:\&quot;source\&quot;,\&quot;propCode\&quot;:\&quot;SPICY\&quot;,\&quot;status\&quot;:\&quot;FAILED\&quot;,\&quot;target\&quot;:\&quot;target\&quot;,\&quot;runId\&quot;:\&quot;30147762712\&quot;}&quot;,&quot;span-id&quot;:&quot;spanId&quot;,&quot;flow&quot;:&quot;Priority Date&quot;}<br></pre><pre>simple terms:</pre><pre>SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date,<br>       sd_data_date_from, sd_data_date_to, sd_warning_date,<br>       sd_valid_from, sd_valid_to, sd_asof_date,<br>       sd_applicable_date_from, sd_applicable_date_to<br>FROM ymtest.rms_shared_dataset<br>WHERE sd_feature_id = &#39;OYV2&#39;<br>  AND sd_dataset_id = &#39;OVERRIDE_MENU_MODEL_OUT&#39;<br>  AND sd_entity = &#39;SPICY&#39;<br>  AND sd_active<br>ORDER BY sd_run_id DESC<br>LIMIT 1</pre><pre>sd_dataset_path :  s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/</pre><pre> the job is supposed to find certain reference files in the File Catalog, but it fails to locate them.</pre><pre>Path and Resolution<br>From the log, we can see the paths involved:</pre><pre>s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/<br>s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/RMS/</pre><pre>-----------------------------------------</pre><pre>[hadoop@ip-10-215-65-102 ~]$ aws s3api list-objects-v2 --bucket rms-oyty-test-emr --prefix &quot;SPICY&quot; | jq -r &#39;.Contents[] | select(.Key | test(&quot;.*/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT&quot;)) | .Key&#39; | sort -r | head -n 2<br>SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/_SUCCESS<br>SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/part-00000-6b7aaf06-3df1-434e-b4c2-ddc0c53bca49-c000.csv</pre><pre>val df = spark.read.csv(SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/part-00000-6b7aaf06-3df1-434e-b4c2-ddc0c53bca49-c000.csv)</pre><pre>--------------------------------------------modification----------------------------------------------------------------------------<br></pre><pre>   SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date,<br>       sd_data_date_from, sd_data_date_to, sd_warning_date,<br>       sd_valid_from, sd_valid_to, sd_asof_date,<br>       sd_applicable_date_from, sd_applicable_date_to<br>FROM ymtest.rms_shared_dataset<br>WHERE sd_feature_id = &#39;OYV2&#39;<br>  AND sd_dataset_id = &#39;OVERRIDE_MENU_MODEL_OUT&#39;<br>  AND sd_entity = &#39;SPICY&#39;<br>  AND sd_active<br>ORDER BY sd_run_id DESC<br>LIMIT 1</pre><pre>previous path :<br>s3://rms-oyty-test-emr/OYV2/OVERRIDE_MENU_PROCESS/30063177330/SPICY/OVERRIDE_MENU_MODEL_OUT/<br></pre><pre>previous runid :<br>30063177330</pre><pre>new path :</pre><pre>s3://rms-oyty-test-emr/SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/</pre><pre>new runid: 20211002001</pre><pre>======================================================failure-restart-----------------------------------------------------------</pre><pre>oozie job -rerun 0054793-241001140631102-oozie-oozi-W -Doozie.wf.rerun.failnodes=true</pre><pre>oozie job -rerun 0005352-241003041356914-oozie-oozi-W -Doozie.wf.rerun.failnodes=true<br></pre><pre>--------------------------after uploading the file -------------------------------------------------------------------------------------------------</pre><pre>Caused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-09-05, 2023-07-27, 1) already exists.<br>        at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713) ~[RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar:0.0.1-SNAPSHOT]<br>        at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401) ~[RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar:0.0.1-SNAPSHOT]<br>        ... 18 more<br>2024-10-01 09:11:34,423 [Executor task launch worker for task 1.0 in stage 13.0 (TID 35)] ERROR org.apache.spark.executor.Executor  - Exception in task 1.0 in stage 13.0 (TID 35)<br>java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (&quot;aurd_prop_code&quot;,&quot;aurd_stay_date&quot;,&quot;aurd_room_collection_id&quot;,&quot;aurd_rem_dem&quot;,&quot;aurd_days_left&quot;,&quot;aurd_run_id&quot;,&quot;aurd_data_date&quot;,&quot;aurd_last_upd_by&quot;,&quot;aurd_last_upd_ts&quot;) VALUES (&#39;SPICY&#39;,&#39;2023-08-24 +00&#39;,1,NULL,27,30147785109,&#39;2023-07-27 +00&#39;,&#39;BATCH&#39;,&#39;2024-10-01 09:11:27.304+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-08-24, 2023-07-27, 1) already exists.  Call getNextException to see other errors in the batch.<br></pre><pre>       --------------------</pre><pre>       query used:</pre><pre>       SELECT * FROM ymtest.act_unc_rem_dem <br>   WHERE &quot;aurd_prop_code&quot; = &#39;SPICY&#39; <br> AND &quot;aurd_stay_date&quot; = &#39;2023-08-24 +00&#39;<br> AND &quot;aurd_data_date&quot; = &#39;2023-07-27 +00&#39;<br> AND &quot;aurd_room_collection_id&quot; = 1;<br></pre><pre>-------------------------------------------------------------------------------</pre><pre> INSERT INTO ymtest.act_unc_rem_dem (<br>  &quot;aurd_prop_code&quot;,<br>  &quot;aurd_stay_date&quot;,<br>  &quot;aurd_room_collection_id&quot;,<br>  &quot;aurd_rem_dem&quot;,<br>  &quot;aurd_days_left&quot;,<br>  &quot;aurd_run_id&quot;,<br>  &quot;aurd_data_date&quot;,<br>  &quot;aurd_last_upd_by&quot;,<br>  &quot;aurd_last_upd_ts&quot;<br>) VALUES (<br>  &#39;SPICY&#39;,<br>  &#39;2023-08-24 +00&#39;,<br>  1,<br>  NULL,<br>  27,<br>  30147785109,<br>  &#39;2023-07-27 +00&#39;,<br>  &#39;BATCH&#39;,<br>  &#39;2024-10-01 09:11:27.304+00&#39;<br>);<br></pre><pre>======================================================input.json============================================================================================<br></pre><pre>s3://rms-oyty-test-emr/SPICY/30147785109/OYV2/ROOM_FORECAST_STLY/CONF/input.json</pre><pre>{<br> &quot;propcode&quot;: &quot;SPICY&quot;,<br> &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br> &quot;spanid&quot;: &quot;2c94cee4c43a4a0a&quot;,<br> &quot;entity&quot;: &quot;SPICY&quot;,<br> &quot;domain&quot;: &quot;PROPERTY&quot;,<br> &quot;inputdatapath&quot;: [<br>  {<br>   &quot;sdsk&quot;: 1100496732,<br>   &quot;runid&quot;: 30147785109,<br>   &quot;datadateto&quot;: &quot;2024-07-26&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-27&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-28&quot;,<br>   &quot;validto&quot;: &quot;2024-07-27 09:06:46.891+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/SPICY/30147785109/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-26 09:06:46.89+00&quot;,<br>   &quot;datasetid&quot;: &quot;TRAN_REM_DMD_BY_DL&quot;<br>  }<br> ],<br> &quot;referencedatapath&quot;: [<br>  {<br>   &quot;referencefileid&quot;: &quot;ROOM_FCST_STLY_CONSTANT&quot;,<br>   &quot;referencepath&quot;: &quot;OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FORECAST_STLY/const.json&quot;<br>  }<br> ],<br> &quot;datadate&quot;: &quot;2024-07-25&quot;,<br> &quot;outputjsonpath&quot;: &quot;SPICY/30147785109/OYV2/ROOM_FORECAST_STLY/CONF/RMS/&quot;,<br> &quot;runid&quot;: 30147785109,<br> &quot;jobid&quot;: &quot;RoomForecastSameTimeLastYear&quot;,<br> &quot;currentdate&quot;: &quot;2024-07-26&quot;,<br> &quot;traceid&quot;: &quot;a9e85bfc61f21d6817ae65dcd2ea128d&quot;,<br> &quot;outputdatapath&quot;: &quot;SPICY/30147785109/OYV2/ROOM_FORECAST_STLY/&quot;,<br> &quot;enddate&quot;: &quot;2025-07-25&quot;,<br> &quot;startdate&quot;: &quot;2024-07-26&quot;<br>}<br></pre><pre>=======NOW LETS BACKTRACK==========================================================================<br>romm forcastreaming demand take 2 file  -&gt; one is current is for run --&gt; 2nd one is the old one<br></pre><pre>merge both of them and that takes input for styl</pre><pre>duplicate: possibility : room_forcast takes two files ....it can be possible that it can be duplicate also<br></pre><pre>this is the path :<br></pre><pre> s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/input.json<br></pre><pre> got two input paths<br>first one for the current eun<br>2nd one is of previous run<br></pre><pre>=================================================input.json=======================================================================<br>{<br> &quot;propcode&quot;: &quot;SPICY&quot;,<br> &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br> &quot;spanid&quot;: &quot;c3ba966066bbbaa5&quot;,<br> &quot;entity&quot;: &quot;SPICY&quot;,<br> &quot;domain&quot;: &quot;PROPERTY&quot;,<br> &quot;inputdatapath&quot;: [<br>  {<br>   &quot;sdsk&quot;: 1100496730,<br>   &quot;runid&quot;: 30147785109,<br>   &quot;datadateto&quot;: &quot;2024-07-26&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-27&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-28&quot;,<br>   &quot;validto&quot;: &quot;2024-07-27 09:02:48.107+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/SPICY/30147785109/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-26 09:02:48.107+00&quot;,<br>   &quot;datasetid&quot;: &quot;STAYDT_DFS_RC_AGG&quot;<br>  },<br>  {<br>   &quot;sdsk&quot;: 1100482303,<br>   &quot;runid&quot;: 30148662105,<br>   &quot;datadateto&quot;: &quot;2024-07-25&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-25&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-27&quot;,<br>   &quot;validto&quot;: &quot;2024-07-26 08:12:46.674+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-25 08:12:46.674+00&quot;,<br>   &quot;datasetid&quot;: &quot;TRAN_REM_DMD_BY_DL&quot;<br>  }<br> ],<br> &quot;referencedatapath&quot;: [<br>  {<br>   &quot;referencefileid&quot;: &quot;ROOM_FCST_REM_DEM_CONSTANT&quot;,<br>   &quot;referencepath&quot;: &quot;OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FCST_REM_DMD/const.json&quot;<br>  }<br> ],<br> &quot;datadate&quot;: &quot;2024-07-25&quot;,<br> &quot;outputjsonpath&quot;: &quot;SPICY/30147785109/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/RMS/&quot;,<br> &quot;runid&quot;: 30147785109,<br> &quot;jobid&quot;: &quot;RoomForecastRemainingDemand&quot;,<br> &quot;currentdate&quot;: &quot;2024-07-26&quot;,<br> &quot;traceid&quot;: &quot;a9e85bfc61f21d6817ae65dcd2ea128d&quot;,<br> &quot;outputdatapath&quot;: &quot;SPICY/30147785109/OYV2/ROOM_FORECAST_REMAINING_DEMAND/&quot;,<br> &quot;enddate&quot;: &quot;2024-07-25&quot;,<br> &quot;startdate&quot;: &quot;2024-07-25&quot;<br>}<br></pre><pre>now comparing the data for current and previous run for the data:</pre><pre>just ran:<br>for run id : 30147785109<br></pre><pre>val df_current_latest_run = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30147785109/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/part-00000-a00a8a64-8ae6-452c-b0a8-baa2134d766f-c000.snappy.parquet&quot;)</pre><pre>df_current_latest_run</pre><pre>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>|SnapShotDate|rba_prop_code|StayDate|rba_bkng_date|rcd_room_collection_id|rpsm_dem_fcst_seg_id|lostierdata|additional_dmd|rba_currency_code|rba_stay_rms|rba_pay_rms|rba_comp_rms|rba_cncl_pay_rms|rba_cncl_pay_rev|rba_cncl_stay_rms|rba_cncl_stay_rev|rba_stay_gross_rev|rba_stay_net_rev|rba_pay_gross_rev|rba_pay_net_rev|rba_total_pay_gross_rev|rba_total_pay_net_rev|rba_total_stay_gross_rev|rba_total_stay_net_rev|net_rms_for_rate|net_rms_for_demand|<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br></pre><pre>val df_previous_latest_run = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/*&quot;)</pre><pre>df_previous_latest_run.show()</pre><pre>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|propCode| rc|dfsId|  stayDate|daysLeft|rooms|additionalDemand|roomsRemaining|additionalDemandRemaining|remainingDemand|paidNetOnTheBooks|paidGrossCancels|cummulativeAdditionalDemand|paidGrossOnTheBooks|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|   NYCFI|  2|    2|2018-05-01|       0|  6.0|             0.0|           6.0|                      0.0|            6.0|             70.0|            78.0|                        0.0|              148.0|<br>|   NYCFI|  1|    3|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|             15.0|             1.0|                        0.0|               16.0|<br>|   NYCFI|  2|    6|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  1|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             0.0|                        0.0|                3.0|<br>|   NYCFI|  2|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  1|    2|2018-05-01|       1|  4.0|             0.0|           6.0|                      0.0|            6.0|             53.0|            61.0|                        0.0|              114.0|<br>|   NYCFI|  2|    2|2018-05-01|       1|  3.0|             0.0|           9.0|                      0.0|            9.0|             66.0|            76.0|                        0.0|              142.0|<br>|   NYCFI|  1|    3|2018-05-01|       1|  1.0|             0.0|           1.0|                      0.0|            1.0|             15.0|             1.0|                        0.0|               16.0|<br>|   NYCFI|  2|    3|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             1.0|                        0.0|                4.0|<br>|   NYCFI|  1|    5|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    5|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    6|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    7|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              2.0|             3.0|                        0.0|                5.0|<br>|   NYCFI|  1|   17|2018-05-01|       1|  2.0|             0.0|           3.0|                      0.0|            3.0|             12.0|             2.0|                        0.0|               14.0|<br>|   NYCFI|  1|    1|2018-05-01|       2|  0.0|             0.0|           1.0|                      0.0|            1.0|             13.0|            12.0|                        0.0|               25.0|<br>|   NYCFI|  2|    1|2018-05-01|       2|  0.0|             0.0|           1.0|                      0.0|            1.0|              7.0|             6.0|                        0.0|               13.0|<br>|   NYCFI|  1|    2|2018-05-01|       2|  3.0|             0.0|           9.0|                      0.0|            9.0|             53.0|            58.0|                        0.0|              111.0|<br>|   NYCFI|  2|    3|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             1.0|                        0.0|                4.0|<br>|   NYCFI|  2|    4|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             3.0|                        0.0|                3.0|<br>|   NYCFI|  1|    5|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br></pre><pre>val df_current_old_run = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30147758923/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/*&quot;)</pre><pre>df_current_old_run.show()<br></pre><pre>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>|SnapShotDate|rba_prop_code|StayDate|rba_bkng_date|rcd_room_collection_id|rpsm_dem_fcst_seg_id|lostierdata|additional_dmd|rba_currency_code|rba_stay_rms|rba_pay_rms|rba_comp_rms|rba_cncl_pay_rms|rba_cncl_pay_rev|rba_cncl_stay_rms|rba_cncl_stay_rev|rba_stay_gross_rev|rba_stay_net_rev|rba_pay_gross_rev|rba_pay_net_rev|rba_total_pay_gross_rev|rba_total_pay_net_rev|rba_total_stay_gross_rev|rba_total_stay_net_rev|net_rms_for_rate|net_rms_for_demand|<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br>+------------+-------------+--------+-------------+----------------------+--------------------+-----------+--------------+-----------------+------------+-----------+------------+----------------+----------------+-----------------+-----------------+------------------+----------------+-----------------+---------------+-----------------------+---------------------+------------------------+----------------------+----------------+------------------+<br></pre><pre>val df_previous_old_run = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/*&quot;)</pre><pre>df_previous_old_run.show()</pre><pre>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|propCode| rc|dfsId|  stayDate|daysLeft|rooms|additionalDemand|roomsRemaining|additionalDemandRemaining|remainingDemand|paidNetOnTheBooks|paidGrossCancels|cummulativeAdditionalDemand|paidGrossOnTheBooks|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|   NYCFI|  2|    2|2018-05-01|       0|  6.0|             0.0|           6.0|                      0.0|            6.0|             70.0|            78.0|                        0.0|              148.0|<br>|   NYCFI|  1|    3|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|             15.0|             1.0|                        0.0|               16.0|<br>|   NYCFI|  2|    6|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  1|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             0.0|                        0.0|                3.0|<br>|   NYCFI|  2|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  1|    2|2018-05-01|       1|  4.0|             0.0|           6.0|                      0.0|            6.0|             53.0|            61.0|                        0.0|              114.0|<br>|   NYCFI|  2|    2|2018-05-01|       1|  3.0|             0.0|           9.0|                      0.0|            9.0|             66.0|            76.0|                        0.0|              142.0|<br>|   NYCFI|  1|    3|2018-05-01|       1|  1.0|             0.0|           1.0|                      0.0|            1.0|             15.0|             1.0|                        0.0|               16.0|<br>|   NYCFI|  2|    3|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             1.0|                        0.0|                4.0|<br>|   NYCFI|  1|    5|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    5|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    6|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  2|    7|2018-05-01|       1|  0.0|             0.0|           0.0|                      0.0|            0.0|              2.0|             3.0|                        0.0|                5.0|<br>|   NYCFI|  1|   17|2018-05-01|       1|  2.0|             0.0|           3.0|                      0.0|            3.0|             12.0|             2.0|                        0.0|               14.0|<br>|   NYCFI|  1|    1|2018-05-01|       2|  0.0|             0.0|           1.0|                      0.0|            1.0|             13.0|            12.0|                        0.0|               25.0|<br>|   NYCFI|  2|    1|2018-05-01|       2|  0.0|             0.0|           1.0|                      0.0|            1.0|              7.0|             6.0|                        0.0|               13.0|<br>|   NYCFI|  1|    2|2018-05-01|       2|  3.0|             0.0|           9.0|                      0.0|            9.0|             53.0|            58.0|                        0.0|              111.0|<br>|   NYCFI|  2|    3|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             1.0|                        0.0|                4.0|<br>|   NYCFI|  2|    4|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             3.0|                        0.0|                3.0|<br>|   NYCFI|  1|    5|2018-05-01|       2|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br></pre><pre>----------------------------------------------------------------------------------------------------------------------------------------------------------------<br></pre><pre>this is input dslist from job config of RommfcstsameTimeLastYearjob which takes the merged input <br></pre><pre>  inputdslist=[{featureid=&quot;OYV2&quot;,datasetid=&quot;TRAN_REM_DMD_BY_DL&quot;,byproperty=&quot;True&quot;,ismandate=&quot;True&quot;,condition=&quot; AND sd_run_id = {runId}&quot;}]</pre><pre>        SELECT *<br>                        FROM ymtest.rms_shared_dataset WHERE sd_feature_id = &#39;OYV2&#39;<br>                        AND sd_dataset_id = &#39;TRAN_REM_DMD_BY_DL&#39;<br>                        AND sd_entity = &#39;SPICY&#39; and sd_run_id=&#39;30147785109&#39;</pre><pre>---------------------------================================================================================----------------------------------------------<br>s3://rms-oyty-test-emr/ROOMFORECASTREMAININGDEMAND/latest/jobconfig_spb.conf<br>                        SELECT *<br>                        FROM ymtest.rms_shared_dataset WHERE sd_feature_id = &#39;OYV2&#39;<br>                        AND sd_dataset_id = &#39;TRAN_REM_DMD_BY_DL&#39;<br>                        AND sd_entity = &#39;SPICY&#39; and sd_run_id=(SELECT max(ds_run_id) FROM ymtest.rms_data_status WHERE ds_prop_code = &#39;SPICY&#39; AND ds_process_action in (&#39;SPB&#39;,&#39;MINI_CONVERSION&#39;) AND ds_active)</pre><pre>                        s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM<br>--------------------------------------------------------------------------------------------------------------------------------------------------</pre><pre>s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/part-00000-c66c5aa9-3230-4e81-ad28-ca095b6ed8b2-c000.snappy.parquet<br></pre><pre>this is the dataframe reading the file noe we need to convert the propcode from NYCFI to SPICY<br>let me know a way to do it</pre><pre>val df2 = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/part-00000-c66c5aa9-3230-4e81-ad28-ca095b6ed8b2-c000.snappy.parquet&quot;)<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|propCode| rc|dfsId|  stayDate|daysLeft|rooms|additionalDemand|roomsRemaining|additionalDemandRemaining|remainingDemand|paidNetOnTheBooks|paidGrossCancels|cummulativeAdditionalDemand|paidGrossOnTheBooks|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br>|   NYCFI|  2|    2|2018-05-01|       0|  6.0|             0.0|           6.0|                      0.0|            6.0|             70.0|            78.0|                        0.0|              148.0|<br>|   NYCFI|  1|    3|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|             15.0|             1.0|                        0.0|               16.0|<br>|   NYCFI|  2|    6|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>|   NYCFI|  1|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              3.0|             0.0|                        0.0|                3.0|<br>|   NYCFI|  2|    8|2018-05-01|       0|  0.0|             0.0|           0.0|                      0.0|            0.0|              0.0|             0.0|                        0.0|                0.0|<br>+--------+---+-----+----------+--------+-----+----------------+--------------+-------------------------+---------------+-----------------+----------------+---------------------------+-------------------+<br></pre><pre>=================================================================================================</pre><pre>import org.apache.spark.sql.functions._</pre><pre>// Read the Parquet file<br>val df_new = spark.read.parquet(&quot;s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/*&quot;)</pre><pre>// Replace &#39;NYCFI&#39; with &#39;SPICY&#39; in the &#39;propCode&#39; column<br>val updatedDf = df_new.withColumn(&quot;propCode&quot;, when(col(&quot;propCode&quot;) === &quot;NYCFI&quot;, &quot;SPICY&quot;).otherwise(col(&quot;propCode&quot;)))</pre><pre>// Show the updated DataFrame</pre><pre>now getting the error :<br></pre><pre> duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-09-05, 2023-07-27, 1) already exists.</pre><pre>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-09-05, 2023-07-27, 1) already exists.<br></pre><pre>{&quot;@timestamp&quot;:&quot;2024-10-02T10:15:30.141Z&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Failed to read/write data due to: Job aborted due to stage failure: Task 3 in stage 23.0 failed 4 times, most recent failure: Lost task 3.3 in stage 23.0 (TID 140) (ip-10-215-65-99.aws-use1.cloud.marriott.com executor 3): java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (\&quot;aurd_prop_code\&quot;,\&quot;aurd_stay_date\&quot;,\&quot;aurd_room_collection_id\&quot;,\&quot;aurd_rem_dem\&quot;,\&quot;aurd_days_left\&quot;,\&quot;aurd_run_id\&quot;,\&quot;aurd_data_date\&quot;,\&quot;aurd_last_upd_by\&quot;,\&quot;aurd_last_upd_ts\&quot;) VALUES (&#39;SPICY&#39;,&#39;2023-09-05 +00&#39;,1,NULL,39,30147799812,&#39;2023-07-27 +00&#39;,&#39;BATCH&#39;,&#39;2024-10-02 10:15:25.881+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-09-05, 2023-07-27, 1) already exists.  Call getNextException to see other errors in the batch.\n\tat org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)\n\tat org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)\n\tat org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:574)\n\tat org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)\n\tat org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:919)\n\tat org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1685)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:713)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1(JdbcUtils.scala:868)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1$adapted(JdbcUtils.scala:867)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2(RDD.scala:1011)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2$adapted(RDD.scala:1011)\n\tat org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2269)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:138)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1516)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(SPICY, 2023-09-05, 2023-07-27, 1) already exists.\n\tat org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)\n\t... 18 more\n\nDriver stacktrace: - Retries left: 1. &quot;,&quot;logger_name&quot;:&quot;com.marriott.elevate.spark.roomforecaststly.job.RoomFcstSameTimeLastYearJob$$anon$1&quot;,&quot;thread_name&quot;:&quot;Driver&quot;,&quot;level&quot;:&quot;INFO&quot;,&quot;level_value&quot;:20000,&quot;app&quot;:&quot;Revenue Management&quot;,&quot;trace-id&quot;:&quot;&quot;,&quot;project&quot;:&quot;RoomForecastSameTimeLastYear&quot;,&quot;customData&quot;:&quot;{\&quot;source\&quot;:\&quot;DataFrame\&quot;,\&quot;propCode\&quot;:\&quot;SPICY\&quot;,\&quot;status\&quot;:\&quot;\&quot;,\&quot;target\&quot;:\&quot;DB: act_unc_rem_dem\&quot;,\&quot;runId\&quot;:\&quot;30147799812\&quot;}&quot;,&quot;span-id&quot;:&quot;&quot;,&quot;flow&quot;:&quot;RoomForecastSameTimeLastYear&quot;,&quot;trace_id&quot;:&quot;33c838a3405ce66f5e10d611f7ae5814&quot;,&quot;trace_flags&quot;:&quot;01&quot;,&quot;span_id&quot;:&quot;08693b28c929d202&quot;}</pre><pre>Check Existing Data:<br>Query the ymtest.act_unc_rem_dem table to check if the record with the above key values already exists.<br> sql<br> SELECT * FROM ymtest.act_unc_rem_dem<br> WHERE aurd_prop_code = &#39;SPICY&#39;<br>   AND aurd_stay_date = &#39;2023-09-05&#39;<br>   AND aurd_data_date = &#39;2023-07-27&#39;<br>   AND aurd_room_collection_id = 1;<br></pre><pre>      query used:</pre><pre>       SELECT * FROM ymtest.act_unc_rem_dem <br>   WHERE &quot;aurd_prop_code&quot; = &#39;SPICY&#39; <br> AND &quot;aurd_stay_date&quot; = &#39;2023-08-24 +00&#39;<br> AND &quot;aurd_data_date&quot; = &#39;2023-07-27 +00&#39;<br> AND &quot;aurd_room_collection_id&quot; = 1;</pre><pre> <br>====================DELETE AFTER 1ST SEPT``````````````````````````````````````````````````````````````````````````````````````<br> Delete  From  ymtest.act_unc_rem_dem where aurd_stay_date &gt; &#39;2023-08-31&#39;</pre><pre> delete from ymtest.act_unc_rem_dem where aurd_prop_code in (&#39;SPICY&#39;,&#39;CHIMT&#39;) and aurd_last_upd_ts&gt;=&#39;2024-09-01&#39;;<br></pre><pre>======================================================failure-restart-----------------------------------------------------------</pre><pre>oozie job -rerun 0054793-241001140631102-oozie-oozi-W -Doozie.wf.rerun.failnodes=true</pre><pre>oozie job -rerun 0048473-241001140631102-oozie-oozi-W -Doozie.wf.rerun.failnodes=true<br></pre><pre>sametimelast year </pre><pre>input data duplicate<br></pre><pre>${PROP_CODE}/${RUN_ID}/OYV2/OVERRIDE_MENU_TARGET_LOAD/CONF/input.json<br></pre><pre>aws s3api list-objects-v2 --bucket rms-oyty-test-emr --prefix &quot;SPICY&quot; | jq -r &#39;.Contents[] | select(.Key | test(&quot;.*/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT&quot;)) | .Key&#39; | sort -r | head -n 2<br>SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/_SUCCESS<br>SPICY/20211002001/OYV2/OVERRIDE_MENU_PROCESS/OVERRIDE_MENU_MODEL_OUT/part-00000-6b7aaf06-3df1-434e-b4c2-ddc0c53bca49-c000.csv</pre><pre>===================================================================CHIMT==========================================================================================================</pre><pre> yarn logs -applicationId application_1727252683508_24192 | grep -C 20 &quot;Error&quot;<br></pre><pre> JOB NAME FOR IT GOT FAILED :  ROOM_FORECAST_STLY_SPARK</pre><pre> {&quot;@timestamp&quot;:&quot;2024-09-29T16:17:13.037Z&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Exception in task 4.1 in stage 18.0 (TID 90)&quot;,&quot;logger_name&quot;:&quot;org.apache.spark.executor.Executor&quot;,&quot;thread_name&quot;:&quot;Executor task launch worker for task 4.1 in stage 18.0 (TID 90)&quot;,&quot;level&quot;:&quot;ERROR&quot;,&quot;level_value&quot;:40000,&quot;stack_trace&quot;:&quot;java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (\&quot;aurd_prop_code\&quot;,\&quot;aurd_stay_date\&quot;,\&quot;aurd_room_collection_id\&quot;,\&quot;aurd_rem_dem\&quot;,\&quot;aurd_days_left\&quot;,\&quot;aurd_run_id\&quot;,\&quot;aurd_data_date\&quot;,\&quot;aurd_last_upd_by\&quot;,\&quot;aurd_last_upd_ts\&quot;) VALUES (&#39;CHIMT&#39;,&#39;2023-10-30 +00&#39;,1,16.55,96,30147748258,&#39;2023-07-25 +00&#39;,&#39;BATCH&#39;,&#39;2024-09-29 16:17:07.407+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(CHIMT, 2023-10-30, 2023-07-25, 1) already exists.  Call getNextException to see other errors in the batch.\n\tat org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)\n\tat org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)\n\tat org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:574)\n\tat org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)\n\tat org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:919)\n\tat org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1685)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:713)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1(JdbcUtils.scala:868)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1$adapted(JdbcUtils.scala:867)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2(RDD.scala:1011)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2$adapted(RDD.scala:1011)\n\tat org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2269)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:138)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1516)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(CHIMT, 2023-10-30, 2023-07-25, 1) already exists.\n\tat org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)\n\t... 18 common frames omitted\n&quot;,&quot;mdc.taskName&quot;:&quot;task 4.1 in stage 18.0 (TID 90)&quot;}<br></pre><pre>SELECT * <br>FROM ymtest.act_unc_rem_dem <br>WHERE aurd_prop_code = &#39;CHIMT&#39; <br>  AND aurd_stay_date = &#39;2023-10-30&#39; <br>  AND aurd_data_date = &#39;2023-07-25&#39; <br>  AND aurd_room_collection_id = 1;</pre><pre>  because we are not running miniconversion </pre><pre>  delete the record after 1st sept</pre><pre> oozie:launcher:T=spark:W=DAILY_BATCH_CYCLE_TRACE-30147762723-CHIMT-9ca41adac1a8064853cb24ddacc4a60-9aa3030d53895ac9:A=ROOM_FORECAST_STLY_SPARK:ID=0033150-240927105655159-oozie-oozi-W<br></pre><pre> {&quot;@timestamp&quot;:&quot;2024-09-30T13:32:52.724Z&quot;,&quot;@version&quot;:&quot;1&quot;,&quot;message&quot;:&quot;Lost task 1.0 in stage 28.0 (TID 171) (ip-10-215-65-118.aws-use1.cloud.marriott.com executor 1): java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (\&quot;aurd_prop_code\&quot;,\&quot;aurd_stay_date\&quot;,\&quot;aurd_room_collection_id\&quot;,\&quot;aurd_rem_dem\&quot;,\&quot;aurd_days_left\&quot;,\&quot;aurd_run_id\&quot;,\&quot;aurd_data_date\&quot;,\&quot;aurd_last_upd_by\&quot;,\&quot;aurd_last_upd_ts\&quot;) VALUES (&#39;CHIMT&#39;,&#39;2024-07-04 +00&#39;,1,17.255000000000003,344,30147762723,&#39;2023-07-25 +00&#39;,&#39;BATCH&#39;,&#39;2024-09-30 13:32:47.32+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(CHIMT, 2024-07-04, 2023-07-25, 1) already exists.  Call getNextException to see other errors in the batch.\n\tat org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)\n\tat org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)\n\tat org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:574)\n\tat org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)\n\tat org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:919)\n\tat org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1685)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:713)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1(JdbcUtils.scala:868)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1$adapted(JdbcUtils.scala:867)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2(RDD.scala:1011)\n\tat org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2$adapted(RDD.scala:1011)\n\tat org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2269)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:138)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1516)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint \&quot;cnstr_pk_aurd_1\&quot;\n  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(CHIMT, 2024-07-04, 2023-07-25, 1) already exists.\n\tat org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)\n\tat org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)\n\t... 18 more\n&quot;,&quot;logger_name&quot;:&quot;org.apache.spark.scheduler.TaskSetManager&quot;,&quot;thread_name&quot;:&quot;task-result-getter-0&quot;,&quot;level&quot;:&quot;WARN&quot;,&quot;level_value&quot;:30000}<br></pre><pre>  SELECT * <br>FROM ymtest.act_unc_rem_dem <br>WHERE aurd_prop_code = &#39;CHIMT&#39; <br>  AND aurd_stay_date = &#39;2024-07-04&#39; <br>  AND aurd_data_date = &#39;2023-07-25&#39; <br>  AND aurd_room_collection_id = 1;<br></pre><pre>  ROOM_FORECAST_STLY_SPARK</pre><pre>  the job config and input.json match for transientforecastetl</pre><pre>  job = RoomForecastSameTimeLastYear</pre><pre>    Log Type: stderr<br>          <br>            Log Upload Time: Thu Oct 03 09:33:06 +0000 2024<br>          <br>            Log Length: 540<br>          Dataset Id &#39;TRAN_ROOM_FCST_PRED&#39; not found for property &#39;CHIMT&#39; <br>RMS SHARED DATASET Query = SELECT sd_sk, sd_dataset_path, sd_run_id, sd_run_date,<br>                sd_data_date_from, sd_data_date_to, sd_warning_date,<br>                sd_valid_from, sd_valid_to, sd_asof_date,<br>                sd_applicable_date_from, sd_applicable_date_to<br>                FROM ymtest.rms_shared_dataset WHERE sd_feature_id = &#39;OYV2&#39;<br>                AND sd_dataset_id = &#39;TRAN_ROOM_FCST_PRED&#39;<br>                AND sd_entity = &#39;CHIMT&#39;  AND sd_run_id = 30147802242<br></pre><pre>                SAMPLE OF NYCFI<br></pre><pre>                emr/NYCFI/30146133662/OYV2/ROOM_FORECAST_INFERENCE/NEW_TRAN_ROOM_FCST_PRED/output.parquet<br></pre><pre>                s3://rms-oyty-test-emr/JAXFP/30147812830/OYV2/ROOM_FORECAST_STLY/CONF/RMS</pre><pre>                 SELECT * <br>FROM ymtest.act_unc_rem_dem <br>WHERE aurd_prop_code = &#39;JAXFP&#39; <br>  AND aurd_stay_date = &#39;2023-08-18&#39; <br>  AND aurd_data_date = &#39;2023-07-27&#39; <br>  AND aurd_room_collection_id = 1;</pre><pre>                Caused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-08-18, 2023-07-27, 1) already exists.<br>        at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)<br>        at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)<br>        ... 18 more</pre><pre>2024-10-04 13:13:08,903 [task-result-getter-3] WARN  org.apache.spark.scheduler.TaskSetManager  - Lost task 0.0 in stage 18.0 (TID 77) (ip-10-215-65-72.aws-use1.cloud.marriott.com executor 1): java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (&quot;aurd_prop_code&quot;,&quot;aurd_stay_date&quot;,&quot;aurd_room_collection_id&quot;,&quot;aurd_rem_dem&quot;,&quot;aurd_days_left&quot;,&quot;aurd_run_id&quot;,&quot;aurd_data_date&quot;,&quot;aurd_last_upd_by&quot;,&quot;aurd_last_upd_ts&quot;) VALUES (&#39;JAXFP&#39;,&#39;2023-08-14 +00&#39;,1,40.188,17,30147812830,&#39;2023-07-27 +00&#39;,&#39;BATCH&#39;,&#39;2024-10-04 13:13:05.515+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-08-14, 2023-07-27, 1) already exists.  Call getNextException to see other errors in the batch.<br>        at org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)<br>        at org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)<br>        at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)<br>        at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:574)<br>        at org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)<br>        at org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:919)<br>        at org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1685)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:713)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1(JdbcUtils.scala:868)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1$adapted(JdbcUtils.scala:867)<br>        at org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2(RDD.scala:1011)<br>        at org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2$adapted(RDD.scala:1011)<br>        at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2269)<br>        at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)<br>        at org.apache.spark.scheduler.Task.run(Task.scala:138)<br>        at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)<br>        at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1516)<br>        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)<br>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)<br>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)<br>        at java.lang.Thread.run(Thread.java:750)<br>Caused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-08-14, 2023-07-27, 1) already exists.<br>        at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)<br>        at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)<br>        ... 18 more<br></pre><pre>merge statement for otp_snapshot<br>code change <br>insert in place of insert merge<br>suresh pilli</pre><pre>2024-10-04 13:13:08,918 [dispatcher-CoarseGrainedScheduler] INFO  org.apache.spark.scheduler.TaskSetManager  - Starting task 0.1 in stage 18.0 (TID 83) (ip-10-215-65-72.aws-use1.cloud.marriott.com, executor 1, partition 0, NODE_LOCAL, 4443 bytes) taskResourceAssignments Map()<br>2024-10-04 13:13:08,918 [task-result-getter-0] WARN  org.apache.spark.scheduler.TaskSetManager  - Lost task 2.0 in stage 18.0 (TID 79) (ip-10-215-65-72.aws-use1.cloud.marriott.com executor 1): java.sql.BatchUpdateException: Batch entry 0 INSERT INTO ymtest.act_unc_rem_dem (&quot;aurd_prop_code&quot;,&quot;aurd_stay_date&quot;,&quot;aurd_room_collection_id&quot;,&quot;aurd_rem_dem&quot;,&quot;aurd_days_left&quot;,&quot;aurd_run_id&quot;,&quot;aurd_data_date&quot;,&quot;aurd_last_upd_by&quot;,&quot;aurd_last_upd_ts&quot;) VALUES (&#39;JAXFP&#39;,&#39;2023-11-11 +00&#39;,1,52.296,106,30147812830,&#39;2023-07-27 +00&#39;,&#39;BATCH&#39;,&#39;2024-10-04 13:13:05.515+00&#39;) was aborted: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-11-11, 2023-07-27, 1) already exists.  Call getNextException to see other errors in the batch.<br>        at org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165)<br>        at org.postgresql.core.ResultHandlerDelegate.handleError(ResultHandlerDelegate.java:52)<br>        at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2402)<br>        at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:574)<br>        at org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:896)<br>        at org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:919)<br>        at org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1685)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:713)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1(JdbcUtils.scala:868)<br>        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$saveTable$1$adapted(JdbcUtils.scala:867)<br>        at org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2(RDD.scala:1011)<br>        at org.apache.spark.rdd.RDD.$anonfun$foreachPartition$2$adapted(RDD.scala:1011)<br>        at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2269)<br>        at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)<br>        at org.apache.spark.scheduler.Task.run(Task.scala:138)<br>        at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)<br>        at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1516)<br>        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)<br>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)<br>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)<br>        at java.lang.Thread.run(Thread.java:750)<br>Caused by: org.postgresql.util.PSQLException: ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;<br>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-11-11, 2023-07-27, 1) already exists.</pre><pre>========================================================================================================================================<br></pre><pre> abets:<br></pre><pre> sTEP 1:  RUN THE JOB<br></pre><pre>created the PR: <a href="https://git.marriott.com/RMS/SimpleETL/pull/77">https://git.marriott.com/RMS/SimpleETL/pull/77</a></pre><pre> wrong location</pre><pre>s3://rms-oyty-test-emr/ABETS/30147793417/OYV2/LOAD_TABLE_TO_S3_BUCKET/TRANS_EVENT_DETAIL/part-00000-626ef50e-1134-48e3-ad94-083be471da4f-c000.snappy.parquet</pre><pre> correct location </pre><pre>s3://bucket-name/ABETS/30147793417/UDEMAND/ADDITIONAL_DEMAND_LOADS_EVENTS_DATA/TRANS_EVENT_DETAIL</pre><pre>wha to do.......</pre><pre>got another wrror in this as well</pre><pre>val sthreeMap = jobConfigurationDetails.dependencymap.get(jobId)</pre><pre>then i changed the whole sthreeMap </pre><pre>jobconfig_additional_demand.conf</pre><pre>s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/jobconfig_additional_demand.conf</pre><pre>s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/AdditionalDemandLoadsEventsData_jobconfig.conf</pre><pre>ADDITIONAL_DEMAND_LOADS_EVENTS_DATA_JOBCONFIG=AdditionalDemandLoadsEventsData_jobconfig.conf<br></pre><pre>spark-submit --verbose --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=4  --driver-memory 1G  --executor-memory 4G  --conf spark.dynamicAllocation.minExecutors=1  --conf spark.dynamicAllocation.maxExecutors=2  --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s  --executor-cores 4   --conf spark.dynamicAllocation.enabled=true  --conf spark.shuffle.service.enabled=true  --deploy-mode cluster  --files s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/connection.conf,s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/jobconfig_additional_demand.conf --class com.marriott.elevate.spark.loadtabletoS3bucket.job.Main  --master yarn s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/LoadTableToS3Bucket-0.0.1-SNAPSHOT.jar  -c connection.conf  -j jobconfig_additional_demand.conf  -i s3://rms-oyty-test-emr/NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/CONF/input.json  -p NYCMQ  -r 30000126911  -k AdditionalDemandLoadsEventsData  -f NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/input.json  -a 2021-10-04  -d 2021-10-03 -t 4d78b5a8f446b989b6d7fa6d578187fd -s 530793e1af513a43<br></pre><pre>24/10/07 12:17:17 ERROR Client: Application diagnostics message: User class threw exception: java.util.NoSuchElementException: key not found: AdditionalDemandLoadsEventsData<br>        at scala.collection.immutable.Map$Map4.apply(Map.scala:437)<br>        at com.marriott.elevate.spark.loadtabletoS3bucket.job.LoadTableToS3BucketJob.&lt;init&gt;(LoadTableToS3BucketJob.scala:34)<br>        at com.marriott.elevate.spark.loadtabletoS3bucket.job.Main$.$anonfun$new$1(Main.scala:33)<br>        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)<br>        at scala.util.Try$.apply(Try.scala:213)<br>        a<br>        at java.lang.reflect.Method.invoke(Method.java:498)<br>        at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:742)</pre><pre>changed -k to LoadTableToS3Bucket</pre><pre>spark-submit --verbose --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=4  --driver-memory 1G  --executor-memory 4G  --conf spark.dynamicAllocation.minExecutors=1  --conf spark.dynamicAllocation.maxExecutors=2  --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s  --executor-cores 4   --conf spark.dynamicAllocation.enabled=true  --conf spark.shuffle.service.enabled=true  --deploy-mode cluster  --files s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/connection.conf,s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/jobconfig_additional_demand.conf --class com.marriott.elevate.spark.loadtabletoS3bucket.job.Main  --master yarn s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/LoadTableToS3Bucket-0.0.1-SNAPSHOT.jar  -c connection.conf  -j jobconfig_additional_demand.conf  -i s3://rms-oyty-test-emr/NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/CONF/input.json  -p NYCMQ  -r 30000126911  -k LoadTableToS3Bucket  -f NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/input.json  -a 2021-10-04  -d 2021-10-03 -t 4d78b5a8f446b989b6d7fa6d578187fd -s 530793e1af513a43</pre><pre>         user: hadoop<br>24/10/07 12:27:15 ERROR Client: Application diagnostics message: User class threw exception: java.util.NoSuchElementException: key not found: LoadTableToS3Bucket<br>        at scala.collection.immutable.Map$Map4.apply(Map.scala:437)<br>        at com.marriott.elevate.spark.loadtabletoS3bucket.job.LoadTableToS3BucketJob.&lt;init&gt;(LoadTableToS3BucketJob.scala:34)<br>        at com.marriott.elevate.spark.loadtabletoS3bucket.job.Main$.$anonfun$new$1(Main.scala:33)<br>        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)<br>        at scala.util.Try$.apply(Try.scala:213)<br>        at com.marriott.elevate.spark.loadtabletoS3bucket.job.Main$.delayedEndpoint$com$marriott$elevate$spark$loadtabletoS3bu<br></pre><pre>spark-submit --verbose --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=4  --driver-memory 1G  --executor-memory 4G  --conf spark.dynamicAllocation.minExecutors=1  --conf spark.dynamicAllocation.maxExecutors=2  --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s  --executor-cores 4   --conf spark.dynamicAllocation.enabled=true  --conf spark.shuffle.service.enabled=true  --deploy-mode cluster  --files s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/connection.conf,s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/jobconfig.conf --class com.marriott.elevate.spark.loadtabletoS3bucket.job.Main  --master yarn s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/LoadTableToS3Bucket-0.0.1-SNAPSHOT.jar  -c connection.conf  -j jobconfig.conf  -i s3://rms-oyty-test-emr/NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/CONF/input.json  -p NYCMQ  -r 30000126911  -k LoadTableToS3Bucket  -f NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/input.json  -a 2021-10-04  -d 2021-10-03 -t 4d78b5a8f446b989b6d7fa6d578187fd -s 530793e1af513a43<br></pre><pre>now changed just the location</pre><pre>spark-submit --verbose --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=4  --driver-memory 1G  --executor-memory 4G  --conf spark.dynamicAllocation.minExecutors=1  --conf spark.dynamicAllocation.maxExecutors=2  --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s  --executor-cores 4   --conf spark.dynamicAllocation.enabled=true  --conf spark.shuffle.service.enabled=true  --deploy-mode cluster  --files s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/connection.conf,s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/jobconfig_additional_demand.conf --class com.marriott.elevate.spark.loadtabletoS3bucket.job.Main  --master yarn s3://rms-oyty-test-emr/LoadTableToS3Bucket/latest/LoadTableToS3Bucket-0.0.1-SNAPSHOT.jar  -c connection.conf  -j jobconfig_additional_demand.conf  -i s3://rms-oyty-test-emr/NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/CONF/input.json  -p NYCMQ  -r 30000126911  -k LoadTableToS3Bucket  -f NYCMQ/30000126911/OYV2/LOAD_TABLE_TO_MDP_S3_BUCKET/input.json  -a 2021-10-04  -d 2021-10-03 -t 4d78b5a8f446b989b6d7fa6d578187fd -s 530793e1af513a43</pre><pre>=========================================================================================================================================================================================================duplicate issue=========================================================================================================================================================================================================</pre><pre>JAXFP/30147812830/OYV2/ROOM_FORECAST_STLY/CONF/RMS<br></pre><pre> s3://rms-oyty-test-emr/JAXFP/30147812830/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/input.json</pre><pre>=========================================================input.json=====================================================================<br>{<br> &quot;propcode&quot;: &quot;JAXFP&quot;,<br> &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br> &quot;spanid&quot;: &quot;8f5083e1ab033baf&quot;,<br> &quot;entity&quot;: &quot;JAXFP&quot;,<br> &quot;domain&quot;: &quot;PROPERTY&quot;,<br> &quot;inputdatapath&quot;: [<br>  {<br>   &quot;sdsk&quot;: 1100500130,<br>   &quot;runid&quot;: 30147812830,<br>   &quot;datadateto&quot;: &quot;2024-07-26&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-27&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-28&quot;,<br>   &quot;validto&quot;: &quot;2024-07-27 16:22:48.179+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/JAXFP/30147812830/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-26 16:22:48.179+00&quot;,<br>   &quot;datasetid&quot;: &quot;STAYDT_DFS_RC_AGG&quot;<br>  },<br>  {<br>   &quot;sdsk&quot;: 14528633,<br>   &quot;runid&quot;: 30148378629,<br>   &quot;datadateto&quot;: &quot;2024-07-25&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-25&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-27&quot;,<br>   &quot;validto&quot;: &quot;2024-07-26 07:05:17.088+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/JAXFP/30148378629/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-25 07:05:17.087+00&quot;,<br>   &quot;datasetid&quot;: &quot;TRAN_REM_DMD_BY_DL&quot;<br>  }<br> ],<br> &quot;referencedatapath&quot;: [<br>  {<br>   &quot;referencefileid&quot;: &quot;ROOM_FCST_REM_DEM_CONSTANT&quot;,<br>   &quot;referencepath&quot;: &quot;OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FCST_REM_DMD/const.json&quot;<br>  }<br> ],<br> &quot;datadate&quot;: &quot;2024-07-25&quot;,<br> &quot;outputjsonpath&quot;: &quot;JAXFP/30147812830/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/RMS/&quot;,<br> &quot;runid&quot;: 30147812830,<br> &quot;jobid&quot;: &quot;RoomForecastRemainingDemand&quot;,<br> &quot;currentdate&quot;: &quot;2024-07-26&quot;,<br> &quot;traceid&quot;: &quot;ee5de196997532ce6e7d5a643e298b19&quot;,<br> &quot;outputdatapath&quot;: &quot;JAXFP/30147812830/OYV2/ROOM_FORECAST_REMAINING_DEMAND/&quot;,<br> &quot;enddate&quot;: &quot;2024-07-25&quot;,<br> &quot;startdate&quot;: &quot;2024-07-21&quot;<br>}<br></pre><pre>=======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================</pre><pre>jobconfigurationdetails {<br>  isstateful = false<br>  flushstateoncomplete = false<br>  jobname = &quot;RoomForecastRemainingDemand&quot;<br>  jobidentifier = &quot;RoomForecastRemainingDemand&quot;<br>  sparkpostproc=true<br>  sparkpreproc=true<br>  inputdslist=[{featureid=&quot;OYV2&quot;,datasetid=&quot;STAYDT_DFS_RC_AGG&quot;,byproperty=&quot;True&quot;,ismandate=&quot;True&quot;,condition=&quot; AND sd_run_id = {runId}&quot;},<br>               {featureid=&quot;OYV2&quot;,datasetid=&quot;TRAN_REM_DMD_BY_DL&quot;,byproperty=&quot;True&quot;,ismandate=&quot;True&quot;,condition=&quot; AND sd_run_id = (SELECT max(ds_run_id) FROM {schema}.rms_data_status WHERE ds_prop_code = &#39;{prop_code}&#39; AND ds_process_action in (&#39;SPB&#39;,&#39;MINI_CONVERSION&#39;) AND ds_active)&quot;}]<br>  preprocessmap {<br>    outputdatapath=&quot;OYV2/ROOM_FORECAST_REMAINING_DEMAND&quot;<br>    outputjsonpath=&quot;CONF/RMS&quot;<br>    referencedata=&quot;True&quot;<br>    domain=&quot;PROPERTY&quot;<br>    }<br>  lastrundetails={featureid=&quot;OYV2&quot;,datasetid=&quot;TRAN_REM_DMD_BY_DL&quot;,offset=&quot;0&quot;}<br>  filecatalog=[{refdatasetid=&quot;ROOM_FCST_REM_DEM_CONSTANT&quot;,refdatasetpath=&quot;OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FCST_REM_DMD/const.json&quot;}]<br>  harddeletelist = []</pre><pre>  triggerinterval = 10000</pre><pre>  dependencymap {<br>    TRAN_REM_DMD_BY_DL {<br>      dataDateFrom = -2190, // asofdate - 6 years<br>      dataDateTo = 0 // asofdate<br>      applicableFrom = -1460  // asofdate - 4 years<br>      applicableTo = 730  //as of date + 2 years<br>      validFrom = 0 // as of date<br>      validTo = 1 // as of date + 1<br>      warningDays = 1<br>    }</pre><pre>  }</pre><pre>  sqlmap {<br>    roomCollectionDefQuery = &quot;&quot;&quot;(select distinct(rcd_room_collection_id) from schemaNameKey.tableNameKey  where rcd_prop_code in (&#39;propCodeKey&#39;) and rcd_data_flag = &#39;dataFlagKey&#39;) as room_collection_def&quot;&quot;&quot;<br>    ratePlanSegMapQuery = &quot;&quot;&quot;(select distinct(rpsm_dem_fcst_seg_id) from schemaNameKey.tableNameKey where rpsm_prop_code in (&#39;propCodeKey&#39;)) as rate_plan_seg_map&quot;&quot;&quot;<br>  }<br></pre><pre>  schemamap {<br>      validtodatenumber = 1<br>      warningdatenumber = 25<br>      sd_feature_id = &quot;OYV2&quot;<br>      sd_dataset_id = &quot;TRAN_REM_DMD_BY_DL&quot;<br>      rate_plan_seg_map_in = &quot;rate_plan_seg_map&quot;<br>      RMS_RATES = &quot;rms_parameters&quot;<br>      room_collection_def_in = &quot;room_collection_def&quot;<br>      gf_event_dtl = &quot;gf_event_dtl&quot;<br>      reservation_aggregation_dataset_id = &quot;STAYDT_DFS_RC_AGG&quot;<br>      remaining_demand_dataset_id = &quot;TRAN_REM_DMD_BY_DL&quot;<br>    }<br>}</pre><pre>=======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================<br>s3://rms-oyty-test-emr/SPICY/30148390479/OYV2/ROOM_FORECAST_REMAINING_DEMAND/REMDEM/part-00000-c66c5aa9-3230-4e81-ad28-ca095b6ed8b2-c000.snappy.parquet</pre><pre>[{featureid=&quot;OYV2&quot;,datasetid=&quot;TRAN_REM_DMD_BY_DL&quot;,byproperty=&quot;True&quot;,ismandate=&quot;True&quot;,condition=&quot; AND sd_run_id = {runId}&quot;}]    </pre><pre>               {featureid=&quot;OYV2&quot;,datasetid=&quot;TRAN_REM_DMD_BY_DL&quot;,byproperty=&quot;True&quot;,ismandate=&quot;True&quot;,condition=&quot; AND sd_run_id = (SELECT max(ds_run_id) FROM {schema}.rms_data_status WHERE ds_prop_code = &#39;{prop_code}&#39; AND ds_process_action in (&#39;SPB&#39;,&#39;MINI_CONVERSION&#39;) AND ds_active)&quot;}]  </pre><pre>SELECT *</pre><pre>                        FROM ymtest.rms_shared_dataset WHERE sd_feature_id = &#39;OYV2&#39;</pre><pre>                        AND sd_dataset_id = &#39;TRAN_REM_DMD_BY_DL&#39;</pre><pre>                        AND sd_entity = &#39;SPICY&#39; and sd_run_id=(SELECT max(ds_run_id) FROM ymtest.rms_data_status WHERE ds_prop_code = &#39;JAXFP&#39; AND ds_process_action in (&#39;SPB&#39;,&#39;MINI_CONVERSION&#39;) AND ds_active)</pre><pre> sd_dataset_path :   s3://rms-oyty-test-emr/JAXFP/30148378629/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL</pre><pre> =============================================================================================</pre><pre> taking another path :</pre><pre>s3://rms-oyty-test-emr/JAXFP/30147802247/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/input.json</pre><pre> input.json</pre><pre> {<br> &quot;propcode&quot;: &quot;JAXFP&quot;,<br> &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br> &quot;spanid&quot;: &quot;36b99ebe32500f6b&quot;,<br> &quot;entity&quot;: &quot;JAXFP&quot;,<br> &quot;domain&quot;: &quot;PROPERTY&quot;,<br> &quot;inputdatapath&quot;: [<br>  {<br>   &quot;sdsk&quot;: 1100499314,<br>   &quot;runid&quot;: 30147802247,<br>   &quot;datadateto&quot;: &quot;2024-07-26&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-27&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-28&quot;,<br>   &quot;validto&quot;: &quot;2024-07-27 15:59:15.08+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/JAXFP/30147802247/OYV2/STAYDATE_DFS_RC_AGGREGATION/MERGEDDATA/&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-26 15:59:15.08+00&quot;,<br>   &quot;datasetid&quot;: &quot;STAYDT_DFS_RC_AGG&quot;<br>  },<br>  {<br>   &quot;sdsk&quot;: 14528633,<br>   &quot;runid&quot;: 30148378629,<br>   &quot;datadateto&quot;: &quot;2024-07-25&quot;,<br>   &quot;warningdate&quot;: &quot;2024-07-26&quot;,<br>   &quot;asofdate&quot;: &quot;2024-07-25&quot;,<br>   &quot;datadatefrom&quot;: &quot;2018-07-27&quot;,<br>   &quot;validto&quot;: &quot;2024-07-26 07:05:17.088+00&quot;,<br>   &quot;datasetpath&quot;: &quot;s3://rms-oyty-test-emr/JAXFP/30148378629/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL&quot;,<br>   &quot;validfrom&quot;: &quot;2024-07-25 07:05:17.087+00&quot;,<br>   &quot;datasetid&quot;: &quot;TRAN_REM_DMD_BY_DL&quot;<br>  }<br> ],<br> &quot;referencedatapath&quot;: [<br>  {<br>   &quot;referencefileid&quot;: &quot;ROOM_FCST_REM_DEM_CONSTANT&quot;,<br>   &quot;referencepath&quot;: &quot;OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FCST_REM_DMD/const.json&quot;<br>  }<br> ],<br> &quot;datadate&quot;: &quot;2024-07-25&quot;,<br> &quot;outputjsonpath&quot;: &quot;JAXFP/30147802247/OYV2/ROOM_FORECAST_REMAINING_DEMAND/CONF/RMS/&quot;,<br> &quot;runid&quot;: 30147802247,<br> &quot;jobid&quot;: &quot;RoomForecastRemainingDemand&quot;,<br> &quot;currentdate&quot;: &quot;2024-07-26&quot;,<br> &quot;traceid&quot;: &quot;e2d51fa707c6dd01627462198e287bc2&quot;,<br> &quot;outputdatapath&quot;: &quot;JAXFP/30147802247/OYV2/ROOM_FORECAST_REMAINING_DEMAND/&quot;,<br> &quot;enddate&quot;: &quot;2024-07-25&quot;,<br> &quot;startdate&quot;: &quot;2024-07-21&quot;<br>}</pre><pre>INSERT INTO ymtest.act_unc_rem_dem (&quot;aurd_prop_code&quot;,&quot;aurd_stay_date&quot;,&quot;aurd_room_collection_id&quot;,&quot;aurd_rem_dem&quot;,&quot;aurd_days_left&quot;,&quot;aurd_run_id&quot;,&quot;aurd_data_date&quot;,&quot;aurd_last_upd_by&quot;,&quot;aurd_last_upd_ts&quot;) VALUES (&#39;JAXFP&#39;,&#39;2023-11-11 +00&#39;,1,52.296,106,30147812830,&#39;2023-07-27 +00&#39;,&#39;BATCH&#39;,&#39;2024-10-04 13:13:05.515+00&#39;) was aborted: </pre><pre>ERROR: duplicate key value violates unique constraint &quot;cnstr_pk_aurd_1&quot;</pre><pre>  Detail: Key (aurd_prop_code, aurd_stay_date, aurd_data_date, aurd_room_collection_id)=(JAXFP, 2023-11-11, 2023-07-27, 1) already exists.  Call getNextException to see other errors in the batch.</pre><pre>=======================================================AdditionDemandStayDate===============================================================================<br></pre><pre>table name:   dem_additional_demand<br></pre><pre>dad_run_id <br>dad_prop_code<br> dad_room_pool_dnubkwdcmp<br>  dad_rpgm_dnubkwdcmp <br>  dad_stay_date<br>   dad_add_dmd dad_last_upd_by<br>    dad_last_upd_ts dad_inventory_type<br>     dad_rate_plan<br>      dad_price_plan<br>20221104001 BAHER TBAP BK2A     11/4/2022 0.279 BATCH 2023-01-23 22:53:00.561 +0530 TBAP BK2A BK2A<br></pre><pre>package com.marriott.elevate.spark.admndstaydate.helper</pre><pre>import com.marriott.elevate.spark.admndstaydate.entity.DemAdditionalDemandRow</pre><pre>import java.sql.{Date, Timestamp}<br>import slick.driver.PostgresDriver.api._<br>import slick.lifted.ProvenShape<br>class AdditionalDemandStayDateConversionSlick(_tableTag: Tag, schemaName: Option[String], tableName: String)<br>  extends Table[DemAdditionalDemandRow](_tableTag, schemaName, tableName){<br>  override def * : ProvenShape[DemAdditionalDemandRow] = (dadRunId, dadPropCode, dadStayDate, dadAddDmd, dadRatePlan,<br>    dadPricePlan, dadInventoryType, dadLastUpdBy, dadLastUpdTs) &lt;&gt; (DemAdditionalDemandRow.tupled, DemAdditionalDemandRow.unapply)</pre><pre>  def dadRunId = column[Long](&quot;dad_run_id&quot;, O.PrimaryKey)                                  int<br>  def dadPropCode = column[String](&quot;dad_prop_code&quot;, O.PrimaryKey)                      varchar<br>  def dadStayDate = column[Date](&quot;dad_stay_date&quot;, O.PrimaryKey)                           Date<br>  def dadAddDmd = column[Double](&quot;dad_add_dmd&quot;, O.SqlType(&quot;NUMERIC&quot;))<br>  def dadRatePlan = column[String](&quot;dad_rate_plan&quot;, O.PrimaryKey)<br>  def dadPricePlan = column[String](&quot;dad_price_plan&quot;, O.PrimaryKey)<br>  def dadInventoryType = column[String](&quot;dad_inventory_type&quot;, O.PrimaryKey)<br>  def dadLastUpdBy = column[String](&quot;dad_last_upd_by&quot;)<br>  def dadLastUpdTs = column[Timestamp](&quot;dad_last_upd_ts&quot;)<br>}<br></pre><pre>case class DemAdditionalDemandRow(<br>                                   dadRunId: Long,<br>                                   dadPropCode: String,<br>                                   dadStayDate: Date,<br>                                   dadAddDmd: Double,<br>                                   dadRatePlan: String,<br>                                   dadPricePlan: String,<br>                                   dadInventoryType: String,<br>                                   dadLastUpdBy: String,<br>                                   dadLastUpdTs: Timestamp<br>                                 )</pre><pre>==============================STYL===============================================================================================<br></pre><pre>act_unc_rem_dem<br></pre><pre>aurd_prop_code aurd_stay_date aurd_data_date aurd_room_pool_coll_id_dnubkwdcmp aurd_rem_dem aurd_days_left aurd_run_id aurd_last_upd_by aurd_last_upd_ts aurd_room_collection_id <br>NYCMQ 12/12/2023 9/6/2023 -999  96 30147679350 BATCH 2024-09-05 21:40:02.962 +0530 1 <br></pre><pre>package com.marriott.elevate.spark.roomforecaststly.helper<br>import com.marriott.elevate.spark.roomforecaststly.entity.RoomFcstSTLY</pre><pre>import java.sql.{Date, Timestamp}<br>import slick.driver.PostgresDriver.api._<br>import slick.lifted.ProvenShape<br>class RoomFcstSameTimeLastYearSlick(_tableTag: Tag, schemaName: Option[String], tableName: String)<br>  extends Table[RoomFcstSTLY](_tableTag, schemaName, tableName){<br>  override def * : ProvenShape[RoomFcstSTLY] = (aurdPropCode, aurdStayDate, aurdDataDate, aurdRemDem, aurdDaysLeft,<br>    aurdRunId, aurdLastUpdBy, aurdLastUpdTs, aurdRoomCollectionId) &lt;&gt; (RoomFcstSTLY.tupled, RoomFcstSTLY.unapply)</pre><pre>  def aurdPropCode = column[String](&quot;aurd_prop_code&quot;, O.PrimaryKey)<br>  def aurdStayDate = column[Date](&quot;aurd_stay_date&quot;, O.PrimaryKey)<br>  def aurdDataDate = column[Date](&quot;aurd_data_date&quot;, O.PrimaryKey)<br>  def aurdRemDem = column[Double](&quot;aurd_rem_dem&quot;, O.SqlType(&quot;NUMERIC&quot;))<br>  def aurdDaysLeft = column[Long](&quot;aurd_days_left&quot;)<br>  def aurdRunId = column[Long](&quot;aurd_run_id&quot;)<br>  def aurdLastUpdBy = column[String](&quot;aurd_last_upd_by&quot;)<br>  def aurdLastUpdTs = column[Timestamp](&quot;aurd_last_upd_ts&quot;)<br>  def aurdRoomCollectionId = column[Long](&quot;aurd_room_collection_id&quot;,O.PrimaryKey)<br>}<br>}</pre><pre>case class RoomFcstSTLY(<br>                                   aurdPropCode: String,<br>                                   aurdStayDate: Date,<br>                                   aurdDataDate: Date,<br>                                   aurdRemDem: Double,<br>                                   aurdDaysLeft: Long,<br>                                   aurdRunId: Long,<br>                                   aurdLastUpdBy: String,<br>                                   aurdLastUpdTs: Timestamp,<br>                                   aurdRoomCollectionId: Long<br>                                 )<br></pre><pre>===================================================================================================<br>--jobconfig<br>C:\Users\ajune750\q4-refresh_plannnnnnnnnnnnnnnnnnnnnnnnnnnnn\TransientForecastETL\RoomForecastSameTimeLastYear\src\main\resources\jobconfig.conf<br>--connectionconfig<br>C:\Users\ajune750\q4-refresh_plannnnnnnnnnnnnnnnnnnnnnnnnnnnn\TransientForecastETL\RoomForecastSameTimeLastYear\src\main\resources\connection.conf<br>-i<br>C:\Users\ajune750\q4-refresh_plannnnnnnnnnnnnnnnnnnnnnnnnnnnn\TransientForecastETL\RoomForecastSameTimeLastYear\src\main\resources\input.json<br>--traceId<br>productcatalogtrace<br>--spanId<br>productcatlogspan</pre><pre>set env variables:</pre><pre>AWS_ACCESS_KEY_ID=ASIATG5EICDXSIY6AVQH;<br>AWS_SECRET_ACCESS_KEY=pukpMPI+ShVjiBJjF7KVD2H6Wns/o044NCUrrcl3;<br>AWS_SESSION_TOKEN=IQoJb3JpZ2luX2VjEAAaCXVzLWVhc3QtMSJHMEUCIHhCwy6B2G56jqoe38OeskASpQZ7i0uwSCF61W0qew2XAiEA2wZrvcIdc3ZeXbajBKWAO3Li8nQoz8kWgzWMQTzuwzYqlgMIWRAAGgwyMjA5OTg0MDYzODMiDCXgPV3evG2HyMDr0yrzApnqeuoeaXLNfA24GbrWxMAFHhIeoYG44qe5VhoyO4VOkZ0RcdvhWfe5HHJxKXMsruG+mwrS9SzbEUoTGOsY1Crz6vjKl8mTWeEQTolrCjR5G944LgDA+zSlWVJx6n7HG0tUGR4fGCo2x2KDstSMzt6gbXPbMwrEpxiAZcZWJimZaFCRqP+ViUqd1Hbiv7puEYlnO9JOuRFutG+YX2QghSSOvcl47OB560mHC/pvvlJZGnpy1Y9GkV8WZastOVnBt1UnsXupTpcrxhL6RrBsRiEBAjoL3wfNAcmy8scddLzu2F9T00HXcuSvg163ntLbvm7csf85jCj8V8935QzSzgZ5qjjDI6/AFR89RGwbd0K/h5z7u6P0onaKfjST3XkbR5d3VM1clI72gbSKk4R5lzVja/Mhmd0yT2XjO8kF+trsgAk5pmaAkbukfyQf9fEZt2V9NqiGD8cMswL/ZYbffxLfHBSvjd7hNetgzpQi/FdkBatgMNnomLgGOqYBbnKFtI1gxbfw+9aVdVGTo99EAGsSG0ikUnM7a86Ha5Frg6pslSaVF5/XFDc8C0zGPc2q4JkuX1POzhuw1SyqWc1RUaXytSA5yJVQfAWQslUZ3UUCi0SXARlGFBI+PUIECb9abF37tdroMkncFIQO2fx5U+IqA8ZWrjYWGOyeHE+JH57fr+PQjzlViox0SzU2hXqMysyWpHUgow8+yHLJSfIPsmhnBA\=\=;<br>hadoop.home.dir=C:\Program Files\Hadoop\bin;<br>spark.master=master<br></pre><pre>-Dspark.master=local[*]<br></pre><pre>===============================================================================errors================================================================</pre><pre>none.get in input.json<br></pre><pre>Exception in thread &quot;main&quot; java.nio.file.AccessDeniedException: s3a://rms-oyty-dev-emr/ABETS/30069761354/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL: getFileStatus on s3a://rms-oyty-dev-emr/ABETS/30069761354/OYV2/ROOM_FORECAST_REMAINING_DEMAND/TRAN_REM_DMD_BY_DL: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 36BHSCSSFH546FS4; S3 Extended Request ID: 7Heof7oQ3M4HfPJkIFgydYGLlsjUuM8KfX0P+h0sqiU4d2KBaNOcaX62bxfyOh7G3dg32l+AGOLcYJF/HeZKpg==; Proxy: null), S3 Extended Request ID: 7Heof7oQ3M4HfPJkIFgydYGLlsjUuM8KfX0P+h0sqiU4d2KBaNOcaX62bxfyOh7G3dg32l+AGOLcYJF/HeZKpg==:403 Forbidden<br> at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:255)<br> at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:175)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3796)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$exists$34(S3AFileSystem.java:4703)<br> at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:499)<br> at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:444)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2337)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2356)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.exists(S3AFileSystem.java:4701)<br> at org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$checkAndGlobPathIfNecessary$4(DataSource.scala:784)</pre><pre>Exception in thread &quot;main&quot; org.apache.hadoop.fs.s3a.AWSBadRequestException: getFileStatus on s3a://rms-oyty-test-emr/OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FORECAST_STLY/const.json: com.amazonaws.services.s3.model.AmazonS3Exception: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; Request ID: 3PRP8SPQXZ2EVGDM; S3 Extended Request ID: b2J3hLsfEoXy6EKMunhKvpGWQfawpTw1rzDQKMvZfz+yElKYP+/C3tB0w8OW+W6F14NxRMOeSA/0hi5eRPMftP6fIpjDNtZh; Proxy: null), S3 Extended Request ID: b2J3hLsfEoXy6EKMunhKvpGWQfawpTw1rzDQKMvZfz+yElKYP+/C3tB0w8OW+W6F14NxRMOeSA/0hi5eRPMftP6fIpjDNtZh:400 Bad Request: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; Request ID: 3PRP8SPQXZ2EVGDM; S3 Extended Request ID: b2J3hLsfEoXy6EKMunhKvpGWQfawpTw1rzDQKMvZfz+yElKYP+/C3tB0w8OW+W6F14NxRMOeSA/0hi5eRPMftP6fIpjDNtZh; Proxy: null)<br> at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:249)<br> at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:175)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3796)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getFileStatus$24(S3AFileSystem.java:3556)<br> at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:499)<br> at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:444)<br> at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2337)<br></pre><pre>Check the S3 Path: <br>Ensure that the S3 path (s3a://rms-oyty-test-emr/OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FORECAST_STLY/const.json) is correct.<br>Verify that the bucket name (rms-oyty-test-emr) and the object key (OozieOrchestrationFlows/CONSTANTS/OYV2_ROOM_FORECAST_STLY/const.json) are correct.</pre><pre> dataframe.select(<br>        col(AurdPropCode),<br>        date_sub(col(RfIStayDate), 364).as(AurdStayDate),<br>        col(AurdRoomCollectionId).cast(Int),<br>        col(AurdRemDem),<br>        col(AurdDaysLeft).cast(Int))<br>      .withColumn(AurdRunId, lit(runId).cast(LongType))<br>      .withColumn(AurdDataDate, getDataDate(date_sub(col(AurdStayDate),1), col(AurdDaysLeft)))<br>      .withColumn(AurdLastUpdBy, lit(Batch))<br>      .withColumn(AurdLastUpdTs, current_timestamp)<br>  }<br></pre><pre>  +--------------+--------------+-----------------------+------------+--------------+-----------+--------------+----------------+--------------------+<br>|aurd_prop_code|aurd_stay_date|aurd_room_collection_id|aurd_rem_dem|aurd_days_left|aurd_run_id|aurd_data_date|aurd_last_upd_by|    aurd_last_upd_ts|<br>+--------------+--------------+-----------------------+------------+--------------+-----------+--------------+----------------+--------------------+<br>|         CLEEL|    2024-04-15|                      1|      57.738|           264|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-12-28|                      1|        20.0|           155|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-08-28|                      2|       6.112|            33|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-10-02|                      2|      28.357|            68|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-03-11|                      2|         5.0|           229|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-03-26|                      3|         1.0|           244|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-11-04|                      2|      11.368|           101|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-12-01|                      2|       9.009|           128|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-10-22|                      2|         9.0|            88|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-03-16|                      3|         1.0|           234|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-04-17|                      3|       3.117|           266|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-09-23|                      1|      53.569|            59|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-12-23|                      1|      27.116|           150|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-07-02|                      2|        12.0|           342|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-04-24|                      2|      15.674|           273|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-11-07|                      3|         1.0|           104|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-07-16|                      3|         2.0|           356|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2023-09-08|                      3|         5.0|            44|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-02-23|                      1|      43.379|           212|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>|         CLEEL|    2024-02-23|                      2|       5.965|           212|30147823307|    2023-07-25|           BATCH|2024-10-09 17:02:...|<br>+--------------+--------------+-----------------------+------------+--------------+-----------+--------------+----------------+--------------------+<br>o<br></pre><pre>        jdbcurl = &quot;jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&quot;</pre><pre>        DbDetails(jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&amp;ApplicationName=RoomForecastSameTimeLastYear,org.postgresql.Driver,appuser,JbjC5nWjQR0nZDvS,50,120000,50000 milliseconds,jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&amp;ApplicationName=RoomForecastSameTimeLastYear,org.postgresql.Driver,appuser,JbjC5nWjQR0nZDvS,5,120000,50000 milliseconds,jdbc:postgresql://rms-oy-test-aurora-postgres-rds-proxy.proxy-clofdwmpxqfq.us-east-1.rds.amazonaws.com:5432/ymdb?prepareThreshold=0&amp;ApplicationName=RoomForecastSameTimeLastYear,org.postgresql.Driver,appuser,JbjC5nWjQR0nZDvS,5,120000,50000 milliseconds)</pre><pre>        sudo spark-submit --conf spark.io.compression.codec=snappy --conf spark.sql.shuffle.partitions=4 --driver-memory 1G --executor-memory 5G --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true --conf spark.yarn.archive=hdfs://ip-10-212-77-33.aws-use1.cloud.marriott.com:8020/user/hadoop/spark_yarn_jars.zip --deploy-mode cluster --files s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/connection.conf,s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/jobconfig.conf --class com.marriott.elevate.spark.roomforecaststly.job.Main --master yarn s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar -c connection.conf -j jobconfig.conf -i s3://rms-oyty-dev-emr/ABETS/30069762769/OYV2/ROOM_FORECAST_STLY/CONF/input.json -p ABETS -r 30069762769 -k RoomForecastSameTimeLastYear -f ABETS/30069762769/OYV2/ROOM_FORECAST_STLY/CONF/input.json -a 2023-09-13 -d 2023-06-14</pre><pre>         sudo spark-submit --conf spark.io.compression.codec=snappy --conf spark.sql.shuffle.partitions=4 --driver-memory 1G --executor-memory 5G --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true --conf spark.yarn.archive=hdfs://ip-10-212-77-33.aws-use1.cloud.marriott.com:8020/user/hadoop/spark_yarn_jars.zip --deploy-mode cluster --files s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/connection.conf,s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/jobconfig.conf --class com.marriott.elevate.spark.roomforecaststly.job.Main --master yarn s3://rms-oyty-dev-emr/RoomForecastSameTimeLastYear/latest/RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar -c connection.conf -j jobconfig.conf -i s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/input.json -p CLEEL -r 30069762770 -k RoomForecastSameTimeLastYear -f CLEEL/30069762770/OYV2/ROOM_FORECAST_STLY/CONF/input.json -a 2023-09-13 -d 2023-06-14</pre><pre>         s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/input.json</pre><pre>         input.json</pre><pre>         -f where to pblish<br>         -i read<br></pre><pre>         final one :</pre><pre>         spark-submit --conf spark.io.compression.codec=snappy --conf spark.sql.shuffle.partitions=4 --driver-memory 1G --executor-memory 5G --conf spark.dynamicAllocation.minExecutors=1 --conf spark.dynamicAllocation.maxExecutors=2 --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s --executor-cores 4 --conf spark.dynamicAllocation.enabled=true --conf spark.yarn.archive=hdfs://ip-10-215-65-102.aws-use1.cloud.marriott.com:8020/user/hadoop/spark_yarn_jars.zip --deploy-mode cluster --files s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/connection.conf,s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/jobconfig.conf --class com.marriott.elevate.spark.roomforecaststly.job.Main --master yarn s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar -c connection.conf -j jobconfig.conf -i s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/input.json -p CLEEL -r 30069762770 -k RoomForecastSameTimeLastYear -f CLEEL/30069762770/OYV2/ROOM_FORECAST_STLY/CONF/input.json -a 2023-09-13 -d 2023-06-14</pre><pre>         spark-submit --verbose --conf spark.io.compression.codec=snappy  --conf spark.sql.shuffle.partitions=4  --driver-memory 1G  --executor-memory 4G  --conf spark.dynamicAllocation.minExecutors=1  --conf spark.dynamicAllocation.maxExecutors=2  --conf spark.dynamicAllocation.schedulerBacklogTimeout=300s  --executor-cores 4   --conf spark.dynamicAllocation.enabled=true  --conf spark.shuffle.service.enabled=true  --deploy-mode cluster --files s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/connection.conf,s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/jobconfig.conf --class com.marriott.elevate.spark.roomforecaststly.job.Main --master yarn s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/RoomForecastSameTimeLastYear-0.0.1-SNAPSHOT.jar -c connection.conf -j jobconfig.conf -i s3://rms-oyty-test-emr/RoomForecastSameTimeLastYear/latest/input.json -t 4d78b5a8f446b989b6d7fa6d578187fd -s 530793e1af513a43</pre><pre>         oozie job -rerun 0003607-241008081420939-oozie-oozi-W -Doozie.wf.rerun.failnodes=true</pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=7e13425e2c7b" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[Interview questions-2025-Backup-5]]></title>
            <link>https://medium.com/@ankitjuneja/interview-questions-2025-92c5726b84c4?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/92c5726b84c4</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Mon, 14 Apr 2025 15:23:32 GMT</pubDate>
            <atom:updated>2025-08-22T04:50:23.962Z</atom:updated>
            <content:encoded><![CDATA[<h3>inventory_test</h3><pre> write unit test case for this using class InventoryTypeDBSinkTest extends Serializable  with AnyFunsuite                                       <br>                             this is  def persistInventoryType(<br>                            inventoryTypeRecord: InventoryTypeDocumentDBRecord,<br>                            itActualCapacity: Int,<br>                            itEffEndTS: Timestamp<br>                          ): Unit = {<br>    try {<br>      log.info(<br>        s&quot;&quot;&quot;Persisting to Inventory Type table for value::<br>           |${inventoryTypeRecord.propertyCode}<br>           $keySeparator${inventoryTypeRecord.inventoryTypeCode}&quot;&quot;&quot;.stripMargin,<br>        CustomloggerMetadata(<br>          &quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          loggerAppName,<br>          jobConfigDetails.jobidentifier,<br>          jobConfigDetails.jobname,<br>          Map.empty[String, String]<br>        )<br>      )<br><br>      val invtTypeUpsert = (InventoryTypeHandler.map(invtType =&gt; (invtType.itPropCode,<br>        invtType.itInventoryType,<br>        invtType.itYieldClass,<br>        invtType.itActiveInd,<br>        invtType.itPremiumInd,<br>        invtType.itBaseOccupancy,<br>        invtType.itMaxExtGuests,<br>        invtType.itEffStartTS,<br>        invtType.itEffEndTS,<br>        invtType.itActualCapacity,<br>        invtType.itRoomClassCode,<br>        invtType.itCategoryCode,<br>        invtType.itInventoryPoolCode,<br>        invtType.itCreatedBy)) +=<br>        (inventoryTypeRecord.propertyCode,<br>          inventoryTypeRecord.inventoryTypeCode,<br>          inventoryTypeRecord.productClassCode,<br>          inventoryTypeRecord.activeInd,<br>          inventoryTypeRecord.premiumInd,<br>          inventoryTypeRecord.baseOccupancy,<br>          inventoryTypeRecord.maximumExtGuests,<br>          new Timestamp(<br>            dateFormat.parse(inventoryTypeRecord.generationTime).getTime<br>          ),<br>          itEffEndTS,<br>          itActualCapacity,<br>          inventoryTypeRecord.roomClassCode,<br>          inventoryTypeRecord.categoryCode,<br>          inventoryTypeRecord.inventoryPoolCode,<br>          inventoryType)<br>        ).transactionally<br><br>      Await.result(<br>        session.database.run(<br>          invtTypeUpsert<br>        ),<br>        10.seconds<br>      )<br><br>      log.info(<br>        s&quot;&quot;&quot;InventoryType value persisted for<br>           |${inventoryTypeRecord.propertyCode}$keySeparator${inventoryTypeRecord}&quot;&quot;&quot;.stripMargin,<br>        CustomloggerMetadata(<br>          &quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          loggerAppName,<br>          jobIdentifiter,<br>          jobName,<br>          Map.empty[String, String]<br>        )<br>      )<br><br>    } catch {<br>      case ex: Exception =&gt;<br>        throw new ExceptionHandler(<br>          ex,<br>          Option(<br>            s&quot;Error occurred persisting Inventory Type values for &quot; +<br>              s&quot;${<br>                inventoryTypeRecord.propertyCode<br>              }$keySeparator${<br>                inventoryTypeRecord.inventoryTypeCode<br>              }&quot;<br>          )<br>        )<br>    }<br>  }<br><br>for InventoryTypeDocumentDBRecord input we have this method<br>def generateInventoryTypeDocumentDBRecord(<br>                                             _id: String = &quot;&quot;,<br>                                             propertyCode: String = &quot;&quot;,<br>                                             inventoryTypeCode: String = &quot;&quot;,<br>                                             productClassCode: String = &quot;&quot;,<br>                                             premiumInd: Boolean = false,<br>                                             baseOccupancy: Int = 0,<br>                                             maximumExtGuests: Int = 0,<br>                                             activeInd: Boolean = false,<br>                                             generationTime: String = &quot;&quot;,<br>                                             roomClassCode: String = &quot;&quot;,<br>                                             categoryCode: String = &quot;&quot;,<br>                                             inventoryPoolCode: String = &quot;&quot;,<br>                                             lastUpdatedBy: String = DbLastUpdatedBy,<br>                                             lastUpdateTime: String = new Timestamp(System.currentTimeMillis()).toString<br>                                           ): InventoryTypeDocumentDBRecord = {<br>    InventoryTypeDocumentDBRecord(<br>      _id,<br>      propertyCode,<br>      inventoryTypeCode,<br>      productClassCode,<br>      premiumInd,<br>      baseOccupancy,<br>      maximumExtGuests,<br>      activeInd,<br>      generationTime,<br>      roomClassCode,<br>      categoryCode,<br>      inventoryPoolCode,<br>      lastUpdatedBy,<br>      lastUpdateTime<br>    )<br>  }<br>  <br>  please use this as our sample function for generating the unit test case for the above code<br>   implicit var jobConfigDetails: JobConfigurationDetails = _<br>  implicit var sparkSession: SparkSession = _<br>  implicit val customerMeta = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, loggerAppName, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, Map(&quot;&quot; -&gt; &quot;&quot;))<br>  var mockDbDetails: DbDetails = _<br>  var mockSession: Session = _<br>  var mockDatabase: Database = _<br>  var mockInventoryCountersHandler: TableQuery[InventoryCountersMapping] = _<br>  var mockInventoryTypeHandler: TableQuery[InventoryTypeMapping] = _<br> // var mockInventoryCountersTableQuery: TableQuery[InventoryCountersMapping] = _<br>  //var mockInventoryTypeTableQuery: TableQuery[InventoryTypeMapping] = _<br>  var inventoryTypeDBSink: InventoryTypeDbSink = _<br><br>  @Before<br>  def setup(): Unit = {<br>    jobConfigDetails = mock[JobConfigurationDetails]<br>   when(jobConfigDetails.jobname).thenReturn(testJobName)<br>  when(jobConfigDetails.jobidentifier).thenReturn(testJobName)<br><br>    mockDbDetails = mock[DbDetails]<br>    mockDatabase = mock[Database]<br>    mockSession = mock[Session]<br><br>    mockInventoryCountersHandler = mock[TableQuery[InventoryCountersMapping]]<br>    mockInventoryTypeHandler = mock[TableQuery[InventoryTypeMapping]]<br><br><br>  //  mockInventoryCountersTableQuery = mock[TableQuery[InventoryCountersMapping]]<br>  //  mockInventoryTypeTableQuery = mock[TableQuery[InventoryTypeMapping]]<br><br>    // Define the expected behavior of the mock implementation<br>    inventoryTypeDBSink = new InventoryTypeDbSink(<br>      mockDbDetails,<br>      &quot;test-schema&quot;,<br>      &quot;test-invt-type&quot;,<br>      &quot;test-invt-conts&quot;<br>    )<br><br>    inventoryTypeDBSink.session = mockSession<br>    when(mockSession.database).thenReturn(mockDatabase)<br><br>  }<br>  //test(&quot;dbSinkOpenTest&quot;) {<br>  @Test<br>  def dbSinkOpenTest(): Unit = {<br>    withObjectMocked[BuilderUtil.type] {<br>      when(<br>        BuilderUtil.prepareInvtCounterTableQuery(<br>          ArgumentMatchers.any(classOf[String]),<br>          ArgumentMatchers.any(classOf[String])<br>        )<br>      ).thenReturn(mockInventoryCountersHandler)<br><br>      when(<br>        BuilderUtil.prepareInvtCounterTableQuery(<br>          ArgumentMatchers.any(classOf[String]),<br>          ArgumentMatchers.any(classOf[String])<br>        )<br>      ).thenReturn(mockInventoryCountersHandler)<br><br>      when(<br>        BuilderUtil.preparePostgresJDCBCDatabase(<br>          ArgumentMatchers.any(classOf[DbDetails])<br>        )<br>      ).thenReturn(mockDatabase)<br><br><br>      //Positive Scenario<br>      when(mockDatabase.createSession()).thenReturn(mockSession)<br>      doNothing().when(mockSession).close()<br><br>      var openResult = inventoryTypeDBSink.open(<br>        DataGenerator.randomLong,<br>        DataGenerator.randomLong()<br>      )<br>      assert(openResult)<br><br>      //Negative Scenario<br>      doThrow(classOf[SlickException]).when(mockDatabase).createSession()<br><br>      intercept[SlickException] {<br>        inventoryTypeDBSink.open(<br>          DataGenerator.randomLong,<br>          DataGenerator.randomLong()<br>        )<br>      }<br>    }<br><br>  }<br><br><br><br>this is generateInventoryCounterDocumentDBRecord method <br> def generateInventoryCounterDocumentDBRecord(<br>      inventoryCounter: InventoryCounter = generateInventoryCounter(),<br>      areRequiredAttributeDifferent: Boolean = false<br>  ): InventoryCounterDocumentDBRecord = {<br>    InventoryCounterDocumentDBRecord(<br>      s&quot;${inventoryCounter.propertyCode}$keySeparator${inventoryCounter.inventoryTypeCode}$keySeparator${inventoryCounter.inventoryDate}&quot;,<br>      inventoryCounter.propertyCode,<br>      inventoryCounter.inventoryDate,<br>      inventoryCounter.inventoryTypeCode,<br>      if (areRequiredAttributeDifferent) faker.number.randomDigit<br>      else inventoryCounter.authorizedCountCounter.head,<br>      if (areRequiredAttributeDifferent) faker.number.randomDigit<br>      else inventoryCounter.actualCountCounter.head,<br>      if (areRequiredAttributeDifferent) faker.number.randomDigit<br>      else inventoryCounter.outOfOrderInventoryTypesCounter.head,<br>      if (areRequiredAttributeDifferent) faker.number.randomDigit<br>      else inventoryCounter.outOfInventoryInventoryTypesCounter.head,<br>      if (areRequiredAttributeDifferent)<br>        DateTimeFormatter<br>          .ofPattern(&quot;yyyy-MM-dd&#39;T&#39;HH:mm:ss.SSS&#39;Z&#39;&quot;)<br>          .format(LocalDateTime.now().plusHours(1))<br>      else inventoryCounter.generationTime<br>    )<br>  }<br>  <br>  which is created as a dummy value method from this case class InventoryTypeDocumentDBRecord(_id: String = &quot;&quot;,<br>                                         propertyCode: String = &quot;&quot;,<br>                                        inventoryTypeCode: String = &quot;&quot;,<br>                                        productClassCode: String = &quot;&quot;,<br>                                        premiumInd: Boolean = false,<br>                                        baseOccupancy: Int = 0,<br>                                         maximumExtGuests: Int = 0,<br>                                         activeInd: Boolean = false,<br>                                         generationTime: String = &quot;&quot;,<br>                                         roomClassCode: String = &quot;&quot;,<br>                                         categoryCode: String = &quot;&quot;,<br>                                         inventoryPoolCode: String = &quot;&quot;, //New attribute<br>                                         lastUpdatedBy: String = DbLastUpdatedBy,<br>                                         lastUpdateTime: String = new Timestamp(System.currentTimeMillis()).toString<br>                                        )<br>          <br>now generate the same dummy value method  generateInventoryCounterDocumentDBRecord for <br> this  case class InventoryCounterDocumentDBRecord(<br>    _id: String = &quot;&quot;,<br>    propertyCode: String = &quot;&quot;,<br>    inventoryDate: String = &quot;&quot;,<br>    inventoryTypeCode: String = &quot;&quot;,<br>    authorizedCountCounter: Int = 0,<br>    actualCountCounter: Int = 0,<br>    outOfOrderInventoryTypesCounter: Int = 0,<br>    outOfInventoryInventoryTypesCounter: Int = 0,<br>    generationTime: String = &quot;&quot;,<br>    lastUpdatedBy: String = DbLastUpdatedBy,<br>    lastUpdateTime: String = new Timestamp(System.currentTimeMillis()).toString<br>)<br><br>keep the dateformat same as the above. you dont need t<br><br><br><br><br><br><br><br><br><br><br><br><br>now <br>          <br>          <br>          <br><br><br> this belongs to thos class class InventoryTypeDBSinkTest extends AnyFunSuite with BeforeAndAfterAll with Serializable {<br> def findActualCapacityforInventoryType(records: Seq[InventoryCountersDocumentDBRecord]): Int = {<br>    implicit val localDateOrdering: Ordering[LocalDate] = Ordering.by(_.toEpochDay)<br>    val latestABC: InventoryCountersDocumentDBRecord = records.<br>      maxBy(record =&gt; LocalDate.parse(record.inventoryDate))<br><br>    latestABC.actualCountCounter<br>  }<br>  <br>  this is case class case class InventoryCountersDocumentDBRecord(_id: String = &quot;&quot;,<br>                                             propertyCode: String,<br>                                             inventoryDate: String,<br>                                             inventoryTypeCode: String,<br>                                             authorizedCountCounter: Int,<br>                                             actualCountCounter: Int,<br>                                             outOfOrderInventoryTypesCounter: Int,<br>                                             outOfInventoryInventoryTypesCounter: Int,<br>                                             generationTime: String,<br>                                             lastUpdatedBy: String,<br>                                             lastUpdateTime: String<br>                                            )<br>           <br>           <br>           build a unit test case for the same and avoid taking date in parameter<br>           <br>           <br>           <br>           <br>           <br>           <br>           --------------solution-------------<br>           this is the unit test code  test(&quot;getInventoryTypeCode should return the inventoryTypeCode of the first record&quot;) {<br>    // Arrange<br>//    val records = Seq(<br>//      DataGenerator.generateInventoryCountersDocumentDBRecord(inventoryTypeCode = &quot;Type1&quot;),<br>//      DataGenerator.generateInventoryCountersDocumentDBRecord(inventoryTypeCode = &quot;Type2&quot;),<br>//      DataGenerator.generateInventoryCountersDocumentDBRecord(inventoryTypeCode = &quot;Type3&quot;)<br>//    )<br><br>  val records = Seq(<br>      InventoryCountersDocumentDBRecord(<br>        _id = &quot;1&quot;,<br>        propertyCode = &quot;PC1&quot;,<br>        inventoryDate = &quot;2022-01-01&quot;,<br>        inventoryTypeCode = &quot;ITC1&quot;,<br>        authorizedCountCounter = 10,<br>        actualCountCounter = 15,<br>        outOfOrderInventoryTypesCounter = 2,<br>        outOfInventoryInventoryTypesCounter = 3,<br>        generationTime = &quot;2022-01-01T00:00:00Z&quot;,<br>        lastUpdatedBy = &quot;user1&quot;,<br>        lastUpdateTime = &quot;2022-01-01T00:00:00Z&quot;<br>      ),<br>      InventoryCountersDocumentDBRecord(<br>        _id = &quot;2&quot;,<br>        propertyCode = &quot;PC2&quot;,<br>        inventoryDate = &quot;2022-02-02&quot;,<br>        inventoryTypeCode = &quot;ITC2&quot;,<br>        authorizedCountCounter = 20,<br>        actualCountCounter = 25,<br>        outOfOrderInventoryTypesCounter = 4,<br>        outOfInventoryInventoryTypesCounter = 5,<br>        generationTime = &quot;2022-02-02T00:00:00Z&quot;,<br>        lastUpdatedBy = &quot;user2&quot;,<br>        lastUpdateTime = &quot;2022-02-02T00:00:00Z&quot;<br>      )<br>    )<br>  and this is the error \inventorytypedefinitiontest\datasource\InventoryTypeDBSinkTest.scala:251:84<br>recursive value records needs type<br>    val inventoryTypeCode = inventoryTypeDBSink.findActualCapacityforInventoryType(records)<br> <br> <br> <br> <br> The error message recursive value records needs type suggests that you are using the records value within its own definition. It could be a problem with the code structure or an issue with the specific test case where this error appears.<br><br>Without seeing the entire code, it&#39;s hard to pinpoint the exact issue, but make sure that records isn&#39;t being referenced within its own initialization. <br><br>From the code snippet you provided, it seems you are trying to use records in the function findActualCapacityforInventoryType call. But if this call is inside the definition of records itself, this would lead to a recursion error.<br><br>Make sure your test is structured in a way that records is fully defined before it&#39;s used in the function call. Here is the correct structure:<br><br>test(&quot;getInventoryTypeCode should return the inventoryTypeCode of the first record&quot;) {<br>  // Arrange<br>  val records = Seq(<br>    InventoryCountersDocumentDBRecord(<br>      // data<br>    ),<br>    InventoryCountersDocumentDBRecord(<br>      // data<br>    )<br>  )<br><br>  // Act<br>  val inventoryTypeCode = inventoryTypeDBSink.findActualCapacityforInventoryType(records)<br>  <br>  // Assert<br>  // your assertions here<br>}<br><br><br><br> write unit test case for this using class InventoryTypeDBSinkTest extends Serializable  with AnyFunsuite<br> def persistInventoryTypeForOldRecord(inventoryTypeRecord: InventoryTypeDocumentDBRecord): Unit = {<br><br>    log.info(<br>      s&quot;&quot;&quot;<br>       Starting update for inventory type<br>       existing record: ${inventoryTypeRecord.propertyCode}-${inventoryTypeRecord.inventoryTypeCode}&quot;&quot;&quot;,<br>      CustomloggerMetadata(<br>        traceId,<br>        spanId,<br>        loggerAppName,<br>        jobConfigDetails.jobidentifier,<br>        jobConfigDetails.jobname,<br>        Map.empty[String, String]<br>      ))<br><br>    val itEffStartTS = new Timestamp(<br>      dateFormat.parse(inventoryTypeRecord.generationTime).getTime<br>    )<br>    val subQuery = InventoryTypeHandler.filter(it =&gt;<br>      it.itPropCode === inventoryTypeRecord.propertyCode &amp;&amp;<br>        it.itInventoryType === inventoryTypeRecord.inventoryTypeCode<br>    ).sortBy(_.itInventoryTypeId.desc).take(1).map(_.itInventoryTypeId)<br><br>    val updateQuery = InventoryTypeHandler.filter(_.itInventoryTypeId in subQuery).map(_.itEffEndTS).update(itEffStartTS)<br>    val result = session.database.run(<br>      DBIO.seq(<br>        subQuery.result.headOption, updateQuery<br>      )<br>    )<br>    Await.result(result, Duration(10, &quot;seconds&quot;))<br><br>    result.onComplete {<br>      case Success(_) =&gt; log.info(&quot;InventoryType value updated for &quot;<br>        + inventoryTypeRecord.propertyCode + &quot;-&quot; + inventoryTypeRecord.inventoryTypeCode,<br>        CustomloggerMetadata(<br>          traceId,<br>          spanId,<br>          loggerAppName,<br>          jobConfigDetails.jobidentifier,<br>          jobConfigDetails.jobname,<br>          Map.empty[String, String]<br>        ))<br>      case Failure(e) =&gt; log.error(&quot;Error occurred for InventoryType value updated for &quot;<br>        + inventoryTypeRecord.propertyCode + &quot;-&quot; + inventoryTypeRecord.inventoryTypeCode +<br>        &quot; :: &quot; + e.getMessage, CustomloggerMetadata(<br>        traceId,<br>        spanId,<br>        loggerAppName,<br>        jobConfigDetails.jobidentifier,<br>        jobConfigDetails.jobname,<br>        Map.empty[String, String]<br>      ))<br>    }<br>  }<br>this is case class <br>case class InventoryTypeDocumentDBRecord(_id: String = &quot;&quot;,<br>                                         propertyCode: String = &quot;&quot;,<br>                                         inventoryTypeCode: String = &quot;&quot;,<br>                                         productClassCode: String = &quot;&quot;,<br>                                         premiumInd: Boolean = false,<br>                                         baseOccupancy: Int = 0,<br>                                         maximumExtGuests: Int = 0,<br>                                         activeInd: Boolean = false,<br>                                         generationTime: String = &quot;&quot;,<br>                                         roomClassCode: String = &quot;&quot;,<br>                                         categoryCode: String = &quot;&quot;,<br>                                         inventoryPoolCode: String = &quot;&quot;, //New attribute<br>                                         lastUpdatedBy: String = DbLastUpdatedBy,<br>                                         lastUpdateTime: String = new Timestamp(System.currentTimeMillis()).toString<br>                                        )<br>          <br>          <br>          <br>          <br>val dateFormat = new SimpleDateFormat(UtcTimestampFormat)<br> val UtcTimestampFormat = &quot;yyyy-MM-dd&#39;T&#39;HH:mm:ss&#39;Z&#39;&quot;<br> -----------------------------------<br> here is the sample test case for producing the outpu like this :<br> like when the InventoryTypeHandler is called in the aabove code it should return  mockInventoryTypeHandler and then it can filter and pass the value to subQuery in val subQuery = InventoryTypeHandler.filter(it =&gt;<br>which this  mockInventoryTypeHandler = mock[TableQuery[InventoryTypeMapping]] <br><br><br><br>   <br>  <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> <br> here is the sample test case for producing the outpu like this :<br> like when the InventoryTypeHandler is called in the aabove code it should return  mockInventoryTypeHandler<br>which this  mockInventoryTypeHandler = mock[TableQuery[InventoryTypeMapping]] <br><br>    def dbSinkProcess(): Unit = {<br>      val inventoryCounterDocumentDBRecord =<br>        generateInventoryCounterDocumentDBRecord()<br><br>      inventoryCountersDbSink.InventoryCountersHandler =<br>        prepareInvtCounterTableQuery(<br>          testSchemaName,<br>          testInventoryCountersName<br>        )<br><br>      inventoryCountersDbSink.InventoryTypeHandler = prepareInvtTypeTableQuery(<br>        testSchemaName,<br>        testInventoryTypeName<br>      )<br><br>    when(<br>      mockDatabase.run(<br>        ArgumentMatchers.any(<br>          classOf[<br>            DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]<br>          ]<br>        )<br>      )<br>    ).thenReturn(Future.successful(1))<br><br>    when(<br>      mockDatabase.run(<br>        ArgumentMatchers.any(<br>          classOf[<br>            DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]<br>          ]<br>        )<br>      )<br>    ).thenReturn(Future.successful(1))<br><br>    inventoryCountersDbSink.process(<br>      (inventoryCounterDocumentDBRecord)<br>    )<br>  }<br>  <br>  <br>  <br>  <br>  <br>  <br>  <br>  <br>  <br>  <br>  solution<br>  <br>  <br>  <br>  <br>  <br>  <br>   test(&quot;persistInventoryTypeForOldRecord should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;)<br>   val mockLog = mock[Logger] // replace Logger with the actual type<br><br>    when(mockDatabase.run(ArgumentMatchers.any())).thenReturn(Future.successful(1))<br><br>    // Act<br>    inventoryTypeDBSink.persistInventoryTypeForOldRecord(inventoryTypeRecord)<br><br>    // Assert<br>    verify(mockLog).info(anyString()) // check that a message was logged<br>    verify(mockDatabase).run(ArgumentMatchers.any()) // check that a database operation was performed<br>  }<br>//<br>  Error<br>  <br>  <br>  value thenReturn is not a member of Nothing<br>  <br>  <br>  <br>  solution 2<br>  <br>  test(&quot;persistInventoryTypeForOldRecord should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;)<br>    val mockLog = mock[Logger] // replace Logger with the actual type<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[_, _, _]]))).thenReturn(Future.successful(1))<br><br>    // Act<br>    inventoryTypeDBSink.persistInventoryTypeForOldRecord(inventoryTypeRecord)<br><br>    // Assert<br>    verify(mockLog).info(anyString()) // check that a message was logged<br>    verify(mockDatabase).run(ArgumentMatchers.any(classOf[DBIOAction[_, _, _]])) // check that a database operation was performed<br>  }<br><br>The error message suggests that the type of the argument expected by run method is not matching with the one provided in any(). The run method is expecting a DBIOAction with specific type parameters.<br><br><br>error \inventorytypedefinitiontest\datasource\InventoryTypeDBSinkTest.scala:337:47<br>type mismatch;<br> found   : slick.driver.PostgresDriver.api.DBIOAction[_$1,_$2,_$3] where type _$3, type _$2, type _$1<br>    (which expands to)  slick.dbio.DBIOAction[_$1,_$2,_$3]<br> required: slick.dbio.DBIOAction[?,slick.dbio.NoStream,Nothing]<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[_, _, _]]))).thenReturn(Future.successful(1))<br> <br> <br> The error message suggests that the type of the argument expected by run method is not matching with the one provided in any(). The run method is expecting a DBIOAction with specific type parameters.<br><br>Here&#39;s how you might modify the test:<br><br>other solution<br><br><br><br><br><br><br><br>definition of run<br><br>this is the run method which is causing the issue  final def run[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = runInternal(a, false)<br><br>    private[slick] final def runInternal[R](a: DBIOAction[R, NoStream, Nothing], useSameThread: Boolean): Future[R] =<br>      try runInContext(a, createDatabaseActionContext(useSameThread), false, true)<br>      catch { case NonFatal(ex) =&gt; Future.failed(ex) }<br>   <br>   <br>   <br>   The run method you&#39;ve provided expects a DBIOAction with specific type parameters: R, NoStream, and Nothing.<br><br>Given this, you should adjust the type parameters in the any() method in your test to match these. Since R is a generic type, you can use _ as a placeholder for any type.<br><br>Here&#39;s how you might modify the test:<br><br><br>solution:<br><br>import org.scalatest.funsuite.AnyFunSuite<br>import org.scalatest.Assertions._<br>import org.mockito.Mockito._<br>import org.mockito.ArgumentMatchers._<br>import scala.concurrent.Future<br>import scala.concurrent.ExecutionContext.Implicits.global<br>import slick.dbio.{DBIOAction, NoStream, Effect}<br><br>class InventoryTypeDBSinkTest extends AnyFunSuite {<br>  test(&quot;persistInventoryTypeForOldRecord should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;)<br>    val mockSession = mock[Session] // replace Session with the actual type<br>    val mockDatabase = mock[Database] // replace Database with the actual type<br>    val mockLog = mock[Logger] // replace Logger with the actual type<br>    val inventoryTypeDBSink = new InventoryTypeDBSink {<br>      override val session = mockSession<br>      override val log = mockLog<br>    }<br><br>    when(mockDatabase.run(any(classOf[DBIOAction[_, NoStream, Nothing]]))).thenReturn(Future.successful(1))<br><br>    // Act<br>    inventoryTypeDBSink.persistInventoryTypeForOldRecord(inventoryTypeRecord)<br><br>    // Assert<br>    verify(mockLog).info(anyString()) // check that a message was logged<br>    verify(mockDatabase).run(any(classOf[DBIOAction[_, NoStream, Nothing]])) // check that a database operation was performed<br>  }<br>}<br><br>   <br>   <br>   <br>   <br>   <br>   --------------------------------------------<br>   <br>point number 1 :   this is the main code  where  InventoryTypeHandler is called<br>    def persistInventoryTypeForOldRecord(inventoryTypeRecord: InventoryTypeDocumentDBRecord): Unit = {<br><br>    log.info(<br>      s&quot;&quot;&quot;<br>       Starting update for inventory type<br>       existing record: ${inventoryTypeRecord.propertyCode}-${inventoryTypeRecord.inventoryTypeCode}&quot;&quot;&quot;,<br>      CustomloggerMetadata(<br>        traceId,<br>        spanId,<br>        loggerAppName,<br>        jobConfigDetails.jobidentifier,<br>        jobConfigDetails.jobname,<br>        Map.empty[String, String]<br>      ))<br><br>    val itEffStartTS = new Timestamp(<br>      dateFormat.parse(inventoryTypeRecord.generationTime).getTime<br>    )<br>    val subQuery = InventoryTypeHandler<br>      .filter(<br>        it =&gt;<br>      it.itPropCode === inventoryTypeRecord.propertyCode &amp;&amp;<br>   <br>   <br>   <br>point number 2:    and this is test code where i have incorporated  {override val InventoryTypeHandler = mockInventoryTypeHandler} so that it does not show null when  InventoryTypeHandler is called but its showing mutable version cannot be overriden <br>   test(&quot;persistInventoryTypeForOldRecord should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;,generationTime = &quot;2021-01-01T00:00:00Z&quot;)<br><br>    val mockLog = mock[Logger] // replace Logger with the actual type<br><br>    mockDatabase = mock[Database]<br>    //jobconfigdetails should come before to the object of inventorytypeDbSink<br>    jobConfigDetails = mock[JobConfigurationDetails]<br>    when(jobConfigDetails.jobname).thenReturn(testJobName)<br>    when(jobConfigDetails.jobidentifier).thenReturn(testJobName)<br><br>    val InventoryTypeHandler = mockInventoryTypeHandler<br><br>    inventoryTypeDBSink = new InventoryTypeDbSink(<br>      mockDbDetails,<br>      &quot;test-schema&quot;,<br>      &quot;test-invt-type&quot;,<br>      &quot;test-invt-conts&quot;<br>    ) {override val InventoryTypeHandler = mockInventoryTypeHandler}<br><br><br>    when(mockDatabase.createSession()).thenReturn(mockSession)<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]]))).thenReturn(Future.successful(1))<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]]))).thenReturn(Future.successful(1))<br>   // doNothing().when(mockSession).close()<br><br>solve this issue in the test case .<br><br><br><br><br>solution:<br><br>It seems like you are trying to override a val that is not a member of the InventoryTypeDbSink class. The override keyword can only be used to override members of a parent class or trait. If InventoryTypeHandler is not a member of InventoryTypeDbSink, you cannot override it.<br><br>But if InventoryTypeHandler is indeed a member of InventoryTypeDbSink and it&#39;s a var, then you can&#39;t override it because `var`s are mutable and cannot be overridden. <br><br>Instead, you can pass mockInventoryTypeHandler as a parameter to the InventoryTypeDbSink constructor if it has a parameter for InventoryTypeHandler. <br><br>Here&#39;s an example:<br><br>val mockInventoryTypeHandler = mock[TableQuery[InventoryTypeMapping]] // replace TableQuery[InventoryTypeMapping] with the actual type<br><br>val inventoryTypeDBSink = new InventoryTypeDbSink(<br>  mockDbDetails,<br>  &quot;test-schema&quot;,<br>  &quot;test-invt-type&quot;,<br>  &quot;test-invt-conts&quot;,<br>  mockInventoryTypeHandler // pass mockInventoryTypeHandler as a parameter<br>)<br>If InventoryTypeDbSink doesn&#39;t have a parameter for InventoryTypeHandler, you can add one:<br><br>class InventoryTypeDbSink(<br>  dbDetails: DbDetails,<br>  schema: String,<br>  invtType: String,<br>  invtConts: String,<br>  val InventoryTypeHandler: TableQuery[InventoryTypeMapping] // add parameter for InventoryTypeHandler<br>) extends Serializable with AnyFunSuite { ... }<br>Then, in your test, you can pass mockInventoryTypeHandler as an argument when you create an instance of InventoryTypeDbSink.<br><br>val inventoryTypeDBSink = new InventoryTypeDbSink(<br>  mockDbDetails,<br>  &quot;test-schema&quot;,<br>  &quot;test-invt-type&quot;,<br>  &quot;test-invt-conts&quot;,<br>  mockInventoryTypeHandler // pass mockInventoryTypeHandler as an argument<br>)<br>Without the complete code and knowing the exact structure of your classes, it&#39;s hard to give a definitive answer. Please adjust the code as needed.<br><br><br><br><br><br><br><br><br><br><br><br><br><br>------------------------solution----------------<br><br>    (mockInventoryTypeHandler.filter _).expects(*).returning(mockInventoryTypeHandler)<br>    (mockInventoryTypeHandler.sortBy _).expects(*).returning(mockInventoryTypeHandler)<br>    (mockInventoryTypeHandler.take _).expects(1).returning(mockInventoryTypeHandler)<br>    (mockInventoryTypeHandler.map _).expects(*).returning(mockInventoryTypeHandler)<br>    (mockInventoryTypeHandler.update _).expects(*).returning(mockDBIO)<br>    (mockSession.database _).expects().returning(mockDatabase)<br>    (mockDatabase.run _).expects(*).returning(Future.successful(()))<br><br>========================quitting=====================================<br><br><br><br>def persistCounterType(<br>                          inventoryCountersRecord: Seq[InventoryCountersDocumentDBRecord]<br>                        ): Unit = {<br><br>    inventoryCountersRecord.foreach(record =&gt; {<br>      try {<br>        log.info(<br>          s&quot;&quot;&quot;Persisting to Inventory Counter table for value::<br>             |${record.propertyCode}<br>           $keySeparator${record.inventoryTypeCode}&quot;&quot;&quot;.stripMargin,<br>          CustomloggerMetadata(<br>            &quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            loggerAppName,<br>            jobConfigDetails.jobidentifier,<br>            jobConfigDetails.jobname,<br>            Map.empty[String, String]<br>          )<br>        )<br><br>        val currentRow = InventoryCountersTable(<br>          record.propertyCode,<br>          Date.valueOf(record.inventoryDate),<br>          record.inventoryTypeCode,<br>          record.authorizedCountCounter,<br>          record.actualCountCounter,<br>          record.outOfOrderInventoryTypesCounter,<br>          record.outOfInventoryInventoryTypesCounter,<br>          new Timestamp(<br>            DateTime.parse(record.generationTime).getMillis<br>          )<br>        )<br><br>        val invtTypeUpsert =<br>          InventoryCountersHandler.insertOrUpdate(currentRow).transactionally<br><br>        Await.result(<br>          session.database.run(<br>            invtTypeUpsert<br>          ),<br>          10.seconds<br>        )<br><br>        log.info(<br>          s&quot;&quot;&quot;InventoryCounter value persisted for<br>             |${record.propertyCode}$keySeparator${inventoryCountersRecord}&quot;&quot;&quot;.stripMargin,<br>          CustomloggerMetadata(<br>            &quot;traceId&quot;,<br>            &quot;spanId&quot;,<br>            loggerAppName,<br>            jobIdentifiter,<br>            jobName,<br>            Map.empty[String, String]<br>          )<br>        )<br><br>      }<br>      catch {<br>        case ex: Exception =&gt;<br>          throw new ExceptionHandler(<br>            ex,<br>            Option(<br>              s&quot;Error Occurred persisting Inventory counter values for &quot; +<br>                s&quot;${<br>                  record.propertyCode<br>                }$keySeparator${<br>                  record.inventoryTypeCode<br>                }&quot;<br>            )<br>          )<br>      }<br>    })<br><br>  }<br>this is case class  InventoryCountersDocumentDBRecord(_id: String = &quot;&quot;,<br>                                             propertyCode: String,<br>                                             inventoryDate: String,<br>                                             inventoryTypeCode: String,<br>                                             authorizedCountCounter: Int,<br>                                             actualCountCounter: Int,<br>                                             outOfOrderInventoryTypesCounter: Int,<br>                                             outOfInventoryInventoryTypesCounter: Int,<br>                                             generationTime: String,<br>                                             lastUpdatedBy: String,<br>                                             lastUpdateTime: String<br>                                            )<br>           <br>            val keySeparator = &quot;-&quot;<br>            <br>            this is InventoryCountersTable(<br>                                   ic_prop_code: String,<br>                                   ic_stay_date: Date,<br>                                   ic_inventory_type: String,<br>                                   ic_trans_rms_auth: Int,<br>                                   ic_actual_capacity: Int,<br>                                   ic_out_of_order: Int,<br>                                   ic_out_of_inventory: Int,<br>                                   ic_source_upd_ts: Timestamp,<br>                                   ic_last_upd_by: String = DbLastUpdatedBy,<br>                                   ic_last_upd_ts: Timestamp = new Timestamp(System.currentTimeMillis())<br>                                 ) extends Serializable<br>class ExceptionHandler(<br>    errorCause: Throwable,<br>    customErrMsg: Option[String] = None<br>) extends BaseException {<br>  val log = new CustomLogger(this.getClass.getName)<br>  <br>  now replicate exactly the below unittest code , keep everything acoording to the below sample unit test code <br>  <br>  test(&quot;persistInventoryType should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;,generationTime = &quot;2021-01-01T00:00:00Z&quot;)<br>    val itActualCapacity = 1<br>    val itEffEndTS = new Timestamp(System.currentTimeMillis())<br>    val mockDatabase = mock[Database] // replace Database with the actual type<br>    val mockLog = mock[Logger] // replace Logger with the actual type<br><br>    when(mockDatabase.createSession()).thenReturn(mockSession)<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]]))).thenReturn(Future.successful(1))<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]]))).thenReturn(Future.successful(1))<br><br>    // Act<br>    val thrown = intercept[ExceptionHandler] {<br>      inventoryTypeDBSink.persistInventoryType(inventoryTypeRecord, itActualCapacity, itEffEndTS)<br>    }<br> <br> <br> <br> <br> <br> here not sble to resilve this error <br>  <br>  test(&quot;persistCounterType should log a message and update the database&quot;) {<br>    // Arrange<br>  / val inventoryCounterRecord = Seq(InventoryCountersDocumentDBRecord(<br>      propertyCode = &quot;P1&quot;,<br>      inventoryTypeCode = &quot;IT1&quot;,<br>      inventoryDate = &quot;2021-01-01&quot;,<br>      authorizedCountCounter = 10,<br>      actualCountCounter = 5,<br>      outOfOrderInventoryTypesCounter = 2,<br>      outOfInventoryInventoryTypesCounter = 3,<br>      generationTime = &quot;2021-01-01T00:00:00Z&quot;<br>    ))<br><br>    val mockDatabase = mock[Database] // replace Database with the actual type<br>    val mockLog = mock[Logger] // replace Logger with the actual type<br><br>    when(mockDatabase.createSession()).thenReturn(mockSession)<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]]))).thenReturn(Future.successful(1))<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]]))).thenReturn(Future.successful(1))<br><br>    // Act<br>    val thrown = intercept[ExceptionHandler] {<br>      inventoryTypeDBSink.persistCounterType(inventoryCounterRecord)<br>    }<br><br>    // Assert<br>   // thrown.message should include (&quot;Error Occurred persisting Inventory counter values for&quot;)<br>    verify(mockLog, times(2)).info(anyString()) // check that a message was logged<br>    verify(mockDatabase).run(ArgumentMatchers.any()) // check that a database operation was performed<br>  }<br><br><br><br><br><br><br><br><br><br><br>  def persistInventoryType(<br>                            inventoryTypeRecord: InventoryTypeDocumentDBRecord,<br>                            itActualCapacity: Int,<br>                            itEffEndTS: Timestamp<br>                          ): Unit = {<br>    try {<br>      log.info(<br>        s&quot;&quot;&quot;Persisting to Inventory Type table for value::<br>           |${inventoryTypeRecord.propertyCode}<br>           $keySeparator${inventoryTypeRecord.inventoryTypeCode}&quot;&quot;&quot;.stripMargin,<br>        CustomloggerMetadata(<br>          &quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          loggerAppName,<br>          jobConfigDetails.jobidentifier,<br>          jobConfigDetails.jobname,<br>          Map.empty[String, String]<br>        )<br>      )<br><br>      val invtTypeUpsert = (InventoryTypeHandler.map(invtType =&gt; (invtType.itPropCode,<br>        invtType.itInventoryType,<br>        invtType.itYieldClass,<br>        invtType.itActiveInd,<br>        invtType.itPremiumInd,<br>        invtType.itBaseOccupancy,<br>        invtType.itMaxExtGuests,<br>        invtType.itEffStartTS,<br>        invtType.itEffEndTS,<br>        invtType.itActualCapacity,<br>        invtType.itRoomClassCode,<br>        invtType.itCategoryCode,<br>        invtType.itInventoryPoolCode,<br>        invtType.itCreatedBy)) +=<br>        (inventoryTypeRecord.propertyCode,<br>          inventoryTypeRecord.inventoryTypeCode,<br>          inventoryTypeRecord.productClassCode,<br>          inventoryTypeRecord.activeInd,<br>          inventoryTypeRecord.premiumInd,<br>          inventoryTypeRecord.baseOccupancy,<br>          inventoryTypeRecord.maximumExtGuests,<br>          new Timestamp(<br>            dateFormat.parse(inventoryTypeRecord.generationTime).getTime<br>          ),<br>          itEffEndTS,<br>          itActualCapacity,<br>          inventoryTypeRecord.roomClassCode,<br>          inventoryTypeRecord.categoryCode,<br>          inventoryTypeRecord.inventoryPoolCode,<br>          inventoryType)<br>        ).transactionally<br><br>      Await.result(<br>        session.database.run(<br>          invtTypeUpsert<br>        ),<br>        10.seconds<br>      )<br><br>      log.info(<br>        s&quot;&quot;&quot;InventoryType value persisted for<br>           |${inventoryTypeRecord.propertyCode}$keySeparator${inventoryTypeRecord}&quot;&quot;&quot;.stripMargin,<br>        CustomloggerMetadata(<br>          &quot;traceId&quot;,<br>          &quot;spanId&quot;,<br>          loggerAppName,<br>          jobIdentifiter,<br>          jobName,<br>          Map.empty[String, String]<br>        )<br>      )<br><br>    } catch {<br>      case ex: Exception =&gt;<br>        throw new ExceptionHandler(<br>          ex,<br>          Option(<br>            s&quot;Error occurred persisting Inventory Type values for &quot; +<br>              s&quot;${<br>                inventoryTypeRecord.propertyCode<br>              }$keySeparator${<br>                inventoryTypeRecord.inventoryTypeCode<br>              }&quot;<br>          )<br>        )<br>    }<br>  }<br> this is unit test class   test(&quot;persistInventoryType should log a message and update the database&quot;) {<br>    // Arrange<br>    val inventoryTypeRecord = InventoryTypeDocumentDBRecord(propertyCode = &quot;P1&quot;, inventoryTypeCode = &quot;IT1&quot;,generationTime = &quot;2021-01-01T00:00:00Z&quot;)<br>    val itActualCapacity = 1<br>    val itEffEndTS = new Timestamp(System.currentTimeMillis())<br>    val mockDatabase = mock[Database] // replace Database with the actual type<br>    val mockLog = mock[Logger] // replace Logger with the actual type<br><br>    when(mockDatabase.createSession()).thenReturn(mockSession)<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]]))).thenReturn(Future.successful(1))<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]]))).thenReturn(Future.successful(1))<br><br><br>    // Act<br>    inventoryTypeDBSink.persistInventoryType(inventoryTypeRecord, itActualCapacity, itEffEndTS)<br><br>    // Assert<br>    verify(mockLog, times(2)).info(anyString()) // check that a message was logged<br>    verify(mockDatabase).run(ArgumentMatchers.any()) // check that a database operation was performed<br>  }<br><br>please also handle the exception in the above unit test case.<br>this is excception Handler class used in main code <br>class ExceptionHandler(<br>    errorCause: Throwable,<br>    customErrMsg: Option[String] = None<br>) extends BaseException {<br>  val log = new CustomLogger(this.getClass.getName)<br>  pleae handle this as am getting the error    <br>com.marriott.mgpapp.spark.inventorytype.exceptions.ExceptionHandler was thrown.<br>com.marriott.mgpapp.spark.inventorytype.exceptions.ExceptionHandler<br> at com.marriott.mgpapp.spark.inventorytype.datasource.InventoryTypeDbSink.persistInventoryType(InventoryTypeDBSink.scala:372)<br> at com.marriott.mgpapp.spark.inventorytypedefinitiontest.datasource.InventoryTypeDBSinkTest.$anonfun$new$18(InventoryTypeDBSinkTest.scala:485)<br> at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)<br><br><br>  def generateInventoryTypeDocumentDBRecord(<br>                                             _id: String = &quot;&quot;,<br>                                             propertyCode: String = &quot;&quot;,<br>                                             inventoryTypeCode: String = &quot;&quot;,<br>                                             productClassCode: String = &quot;&quot;,<br>                                             premiumInd: Boolean = false,<br>                                             baseOccupancy: Int = 0,<br>                                             maximumExtGuests: Int = 0,<br>                                             activeInd: Boolean = false,<br>                                             generationTime: String = &quot;&quot;,<br>                                             roomClassCode: String = &quot;&quot;,<br>                                             categoryCode: String = &quot;&quot;,<br>                                             inventoryPoolCode: String = &quot;&quot;,<br>                                             lastUpdatedBy: String = DbLastUpdatedBy,<br>                                             lastUpdateTime: String = new Timestamp(System.currentTimeMillis()).toString<br>                                           ): InventoryTypeDocumentDBRecord = {<br>    InventoryTypeDocumentDBRecord(<br>      _id,<br>      propertyCode,<br>      inventoryTypeCode,<br>      productClassCode,<br>      premiumInd,<br>      baseOccupancy,<br>      maximumExtGuests,<br>      activeInd,<br>      generationTime,<br>      roomClassCode,<br>      categoryCode,<br>      inventoryPoolCode,<br>      lastUpdatedBy,<br>      lastUpdateTime<br>    )<br>  }<br>  <br>  <br>  <br>  <br> solution<br> <br> <br> <br> difference between,<br> <br> val inventoryTypeDBSink = new inventoryTypeDBSink{<br>      override val session = mockSession<br>      override val log = mockLog<br>    }<br> <br> <br> and <br> inventoryTypeDBSink = new InventoryTypeDbSink(<br>      mockDbDetails,<br>      &quot;test-schema&quot;,<br>      &quot;test-invt-type&quot;,<br>      &quot;test-invt-conts&quot;,<br>    //  mockInventoryTypeHandler<br>    )<br> <br> <br> <br> <br> most frequent error:<br> <br> C:\Users\ajune750\RMMGP-27522-inventory-Types\InventoryDefinitionETL\InventoryTypeDefinition\src\test\scala\com\marriott\mgpapp\spark\inventorytypedefinitiontest\datasource\InventoryTypeDBSinkTest.scala:479:52<br>value thenReturn is not a member of Nothing<br>    when(mockDatabase.run(ArgumentMatchers.any())).thenReturn(Future.successful(1))<br>  this when DBIOS in Persistold_record<br>  <br>  this is how it got solved:(persistold_record)<br>  <br>   when(mockDatabase.createSession()).thenReturn(mockSession)<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Int, NoStream, Effect.Write with Effect.Transactional]]))).thenReturn(Future.successful(1))<br>    when(mockDatabase.run(ArgumentMatchers.any(classOf[DBIOAction[Unit, NoStream, Effect.Read with Effect.Write]]))).thenReturn(Future.successful(1))<br>   /<br>   <br>   isme bhi run me change karna padega<br>    when(mockDatabase<br>      .run(ArgumentMatchers<br>        .any()))<br>      .thenReturn(Future.successful(1))<br>   <br>   also we see in the debugger how the values is coming for mock data refrences<br>   scrrenshot in teams<br>   <br>   <br> ------------------------------------------------------  <br>   <br>   now got issue in exception handling<br>   <br>   <br>com.marriott.mgpapp.spark.inventorytype.exceptions.ExceptionHandler was thrown.<br>com.marriott.mgpapp.spark.inventorytype.exceptions.ExceptionHandler<br> at com.marriott.mgpapp.spark.inventorytype.datasource.InventoryTypeDbSink.persistInventoryType(InventoryTypeDBSink.scala:372)<br> at com.marriott.mgpapp.spark.inventorytypedefinitiontest.datasource.InventoryTypeDBSinkTest.$anonfun$new$18(InventoryTypeDBSinkTest.scala:485)<br> at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)</pre><h3>2nd round, Nagarro</h3><p>What are the pain point in the project yo have faced?</p><p>How do you increase the optimization?</p><p>What is the size of data in the parqut file?</p><p>What is singelton in scala?</p><p>Advantages of avro data and its disdvantages?</p><p>Difference between repartition and coalesce?</p><p>WHen do you get out of memory exception?</p><p>Skewed exception?</p><p>Indexing in database?</p><p>How do you use Stored Procedure?</p><p>Explain Subquery?</p><p>APTIVA -2nd round</p><p>MAnaged and external table difference</p><p>difference between partioning and bucketing</p><p>Difference between narrow and wide transformation</p><p>Difeerence between cache and persist</p><p>Spark Streaming</p><p>Java Training</p><p>SCD, Explain type 1 and type 2</p><p>Explain Broker , Partition ?</p><p>Explain Consumer Group and how it works</p><p>HAve you worked with Redis/ Cassandra/HBAse?</p><p>Are you doing increamental load or full load?</p><p>Fully explain delta table functioning’?</p><p>Difference beween Hive and Impala?</p><p>EPAM Interview.</p><p>Explain the recent project , Are you doing the full load or incremental load on daily basis/weekly basis?</p><p>Which on premise cluster have you used?</p><p>Difference between arc/orc/parquet ?</p><p>Difference between scp and distcp ?</p><p>Spark Architecture?</p><p>What if you give the spark submit command and clusters is not allowing?</p><p>What happens when is your name node not working ?</p><p>How do you know that name node is not Working?</p><p>Kafka dataframe? What about schema registry?</p><p>How can you make your kafka message idempotent?</p><p>How do you handle Out of memory Exception?</p><p>Explain Broadcast join?</p><p>Spark memory 10 GB ? How memory is divided inside the executor?</p><p>1 Tb Data file that is needed to be processed by 16 cores 10 gb capacity , will it be able to do it?</p><p>What is lazy evolution in Spark?</p><p>What is explain in sql for a query like Explain Select * from table1?</p><p>In Dataframe to RDD conversion, how sql query executed in this plan ?</p><p>Expalin catalyst optimiser?</p><p>If there are two lage tables of 500 gb what would be the technique used?</p><p>What is Rack Awareness in hadoop?</p><p>What is he role of Deltatable in Databricks?</p><p>Where do you store your data in Deltatable?</p><p>Explain Transaction log?</p><p>Explain the difference in scd specially type 1 and type 2?</p><p>What are the types of joins available in sql?</p><p>Expalin semi and anti join and left anti join with example?</p><p>Difference between Rank and Desnse Rank?</p><p>if table a and table b has 100 records , with joins on these two will the records increase or derease?</p><p>Explain Recursivity?</p><p>Difference between union and union All</p><p>A table has 100 column tell me a approach that how can we take 95 columns, solution is related regarding Exclude?</p><p>s3 Bucket , How long can we revert the data if it is deleted?</p><p>In ec2 how can we mak sure it is not be deleted by anyone?</p><p>difference between EBS and S3 storage?</p><p>Difference between Athena and Glue?</p><p>Difference betwen Serverless and Server?</p><p>If there is change in glue schema what will it do will the glue job fail ?what we can do to ensure the job is not failed?</p><p>if there are 10 files in s , glue job is picking 1of them or crawler is picking 10 of them , how many fikes it will generate ….correct the question as well?</p><p>Sed command, 744 in linux, if i have to make 10 folders in one directory</p><p>change the owner and permission</p><p>Slicing concept in Python</p><p>Can we name the exception or define it</p><p>how to raise exceoption</p><p>difference between return and yield?</p><p>traits, cn we do implementation in this</p><p>val and var</p><p>recursion and tail recursion</p><p>companion object</p><p>how can we make the fike output in one folder…using Repartion and coalesce</p><p>You have hands-on experience with Kafka and Spark for orchestrating pipelines. Could you discuss a specific project where you worked with these technologies to enable real-time data ingestion and processing for streaming applications?</p><h3>high Scalable data pipeline:</h3><pre>“Building and optimizing large-scale data ingestion pipelines” — This is a common responsibility in data engineering. Here&#39;s an explanation in plain terms:<br><br>1. What is a data ingestion pipeline?<br>A data ingestion pipeline is a system or set of processes that collects data from different sources and moves it to a central storage or processing system, like a data warehouse, data lake, or analytics platform.<br><br>Sources might include databases, APIs, flat files (like CSVs), IoT devices, or third-party tools.<br><br>The goal is to make data available for analytics, reporting, machine learning, etc.<br><br>2. What does “large-scale” mean here?<br>It refers to:<br><br>High volume of data (terabytes to petabytes).<br><br>High velocity (real-time or near real-time data).<br><br>Variety (structured, semi-structured, unstructured data).<br><br>Systems need to be reliable, fault-tolerant, and scalable.<br><br>3. What’s involved in “building” a pipeline?<br>It includes:<br><br>Choosing the right tools/technologies (e.g., Apache Kafka, Spark, Flink, Airflow, etc.)<br><br>Writing code to extract, transform, and load data (ETL or ELT).<br><br>Automating workflows using schedulers or orchestration tools.<br><br>Ensuring data quality and validation during ingestion.<br><br>4. What does “optimizing” mean?<br>Once the pipeline is built, optimization involves:<br><br>Improving performance (e.g., faster processing, lower latency).<br><br>Reducing costs (e.g., minimizing compute/storage usage).<br><br>Making it fault-tolerant (e.g., retry mechanisms, error logging).<br><br>Scalability (e.g., handling more data or more users without rewriting the system).</pre><h3>high througput in data:</h3><pre><br>1. Real-time vs Batch Workloads<br>Batch Workloads<br>Data is collected over time and processed in chunks (e.g., every hour or day).<br><br>Example: At midnight, process all sales from the previous day.<br><br>Common tools: Apache Spark, AWS Glue, Dataflow (batch mode), Hive.<br><br>Real-time Workloads<br>Data is ingested and processed as it arrives, with minimal delay.<br><br>Example: Streaming logs from a web app and triggering alerts within seconds if errors spike.<br><br>Common tools: Apache Kafka, Apache Flink, Spark Structured Streaming, Amazon Kinesis, Apache Pulsar.<br><br>2. High-Throughput Data Processing (Real-Time)<br>“High-throughput” means the system can handle a large volume of data events per second/minute without bottlenecks.<br><br>This involves:<br>Efficient streaming frameworks (e.g., Flink, Spark Streaming) that can scale horizontally.<br><br>Partitioning data for parallel processing (e.g., Kafka partitions, Flink parallelism).<br><br>Using message brokers like Kafka or Pulsar that support millions of events per second.<br><br>Implementing backpressure handling so the system doesn’t crash under load.<br><br>Ensuring low-latency processing (i.e., data gets processed within milliseconds to seconds).<br><br>Putting It Together<br>A solid data engineering setup for this might include:<br><br>Kafka/Kinesis for ingestion.<br><br>Flink or Spark Streaming for processing and enrichment.<br><br>A serving layer like Redis, Elasticsearch, or a data lakehouse (like Delta Lake).<br><br>Batch jobs for historical processing or reprocessing (via Airflow, Glue, or Spark).</pre><h3>scalable data storage solutions in the context of cloud infrastructure</h3><pre>Types of Scalable Storage Solutions in the Cloud<br>a. Object Storage (Unstructured Data)<br>AWS S3, Google Cloud Storage, Azure Blob Storage<br><br>Cost-effective, infinite scaling, ideal for logs, raw files, backups, ML datasets.<br><br>b. Data Lakes / Lakehouses<br>Built on top of object storage (e.g., S3) with metadata layers.<br><br>Tools: Databricks Delta Lake, Apache Hudi, Iceberg.<br><br>Supports both batch and streaming workloads.<br><br>c. Data Warehouses (Structured Analytics)<br>Amazon Redshift, Google BigQuery, Snowflake, Azure Synapse.<br><br>Good for SQL analytics, BI dashboards.<br><br>Auto-scaling and separation of storage/compute in some tools (e.g., BigQuery, Snowflake).<br><br>d. NoSQL / Key-Value Stores<br>DynamoDB, Bigtable, Cassandra — good for high-throughput, low-latency use cases.<br><br>Used in real-time analytics, session stores, metadata management.<br><br>e. Distributed File Systems / Block Storage<br>EBS (Elastic Block Store), GCP Persistent Disk, EFS/FSx for shared file systems.<br><br>Less common for data lakes, but useful in certain compute-heavy workloads.<br><br></pre><h3><strong>Given your experience with AWS EMR and setting up CICD pipelines with Jenkins , how do you optimize resource usage for cost-effective performance when deploying Spark jobs on cloud clusters?</strong></h3><pre>Optimization Strategies:<br>1. Right-Sizing EMR Clusters<br>Used spot instances for non-critical nodes (e.g., core/task) to reduce costs by up to 70%.<br><br>Chose optimal instance types (e.g., r5, m5, or i3) based on workload characteristics — e.g., memory-heavy aggregations got r5 nodes.<br><br>Configured auto-scaling policies based on YARN memory and CPU usage metrics, allowing the cluster to scale dynamically during peak and off-peak hours.<br><br>2. Spark Job Tuning<br>Tuned Spark parameters like:<br><br>executorMemory, executorCores, numExecutors to avoid under/over-utilization.<br><br>Used Dynamic Allocation to auto-scale executors based on workload.<br><br>Cached intermediate data when repeatedly accessed and used efficient formats like Parquet with Snappy compression.<br><br>Ensured partition pruning and optimized joins via broadcast hints where applicable.<br><br>3. Efficient Data I/O<br>Read data from S3 using EMRFS with consistent view turned off (when safe) for performance.<br><br>Wrote outputs in partitioned directories and used bucketing to reduce shuffle on joins.<br><br>4. CI/CD with Jenkins<br>Automated deployment of Spark applications via Jenkins pipelines:<br><br>Built and versioned Spark jobs using SBT or Maven.<br><br>Integrated with AWS CLI/EMR API to provision clusters or submit jobs to persistent ones.<br><br>Enabled parameterized builds to control cluster size, job configs, and target datasets for each environment (dev, QA, prod).<br><br>Implemented canary runs for critical workflows to validate data output before full-scale execution.<br><br>5. Cost Monitoring &amp; Governance<br>Integrated CloudWatch, AWS Cost Explorer, and custom Prometheus/Grafana dashboards for tracking EMR costs per job/team.<br><br>Set up lifecycle policies for logs and intermediate data in S3 to reduce storage costs.<br><br>Impact:<br>Reduced EMR compute costs by ~40% without sacrificing SLAs.<br><br>Improved job completion time by 25% through memory and shuffle optimization.<br><br>CI/CD reduced manual deployment time from hours to minutes and ensured consistency across environments.</pre><h3><strong>— — — — — — — — — -Dentsu Interview — — — — — —</strong></h3><h3>Question1:</h3><pre>✅ Question:<br>Given two tables:<br><br>Table 1<br>&#39;A&#39;<br>&#39;B&#39;<br>&#39;C&#39;<br>NULL<br><br><br>Table 2<br><br>&#39;A&#39;<br>&#39;A&#39;<br>&#39;B&#39;<br>NULL<br>NULL<br>Write SQL queries to perform the following joins:<br><br>INNER JOIN<br><br>LEFT JOIN<br><br>RIGHT JOIN<br><br>FULL OUTER JOIN</pre><pre>🔹 1. INNER JOIN<br>sql<br>Copy<br>Edit<br>SELECT * <br>FROM Table1 t1<br>INNER JOIN Table2 t2 <br>ON t1.col = t2.col;<br>✅ Output:<br><br>lua<br>Copy<br>Edit<br>t1.col | t2.col<br>----------------<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;B&#39;     | &#39;B&#39;<br>Only rows where values match in both tables are included. &#39;C&#39; and NULLs are excluded.<br><br>🔹 2. LEFT JOIN<br>sql<br>Copy<br>Edit<br>SELECT * <br>FROM Table1 t1<br>LEFT JOIN Table2 t2 <br>ON t1.col = t2.col;<br>✅ Output:<br><br>sql<br>Copy<br>Edit<br>t1.col | t2.col<br>----------------<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;B&#39;     | &#39;B&#39;<br>&#39;C&#39;     | NULL<br>NULL    | NULL<br>NULL    | NULL<br>All rows from Table1 are included. Matches from Table2 are shown; unmatched rows have NULL.<br><br>🔹 3. RIGHT JOIN<br>sql<br>Copy<br>Edit<br>SELECT * <br>FROM Table1 t1<br>RIGHT JOIN Table2 t2 <br>ON t1.col = t2.col;<br>✅ Output:<br><br>sql<br>Copy<br>Edit<br>t1.col | t2.col<br>----------------<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;B&#39;     | &#39;B&#39;<br>NULL    | NULL<br>NULL    | NULL<br>All rows from Table2 are included. Matches from Table1 shown; unmatched ones get NULL.<br><br>🔹 4. FULL OUTER JOIN<br>sql<br>Copy<br>Edit<br>SELECT * <br>FROM Table1 t1<br>FULL OUTER JOIN Table2 t2 <br>ON t1.col = t2.col;<br>✅ Output:<br><br>sql<br>Copy<br>Edit<br>t1.col | t2.col<br>----------------<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;A&#39;     | &#39;A&#39;<br>&#39;B&#39;     | &#39;B&#39;<br>&#39;C&#39;     | NULL<br>NULL    | NULL<br>NULL    | NULL<br>All data from both tables included. Matching where possible, NULL where not.</pre><h3><strong>Question 2:</strong></h3><h3><strong>Q: What is the purpose of the </strong><strong>DUAL table in Oracle, and what is the data type of the literal </strong><strong>1 in the query </strong><strong>SELECT 1 FROM DUAL? Explain in detail.</strong></h3><pre>DUAL is a dummy table used when no actual table is needed.<br>DUAL is a special one-row, one-column table in Oracle.<br><br>SELECT 1 FROM DUAL returns the constant 1.<br><br>The data type of 1 is NUMBER in Oracle.</pre><h3><strong>Question 3: NVL AND NVL2 IN ORACLE SQL</strong></h3><pre>These are Oracle SQL functions used to handle NULL values.<br><br>1. NVL(expr1, expr2)<br>Replaces NULL with another value.<br><br>If expr1 is NULL, it returns expr2.<br>Otherwise, it returns expr1.<br><br>Example:<br>sql<br>Copy<br>Edit<br>SELECT NVL(NULL, &#39;Fallback&#39;) FROM DUAL;<br>-- Output: Fallback<br><br>SELECT NVL(&#39;Actual&#39;, &#39;Fallback&#39;) FROM DUAL;<br>-- Output: Actual<br>2. NVL2(expr1, expr2, expr3)<br>If expr1 is not NULL, returns expr2.<br>If expr1 is NULL, returns expr3.<br><br>So it&#39;s a bit more flexible than NVL.<br><br>Example:<br>sql<br>Copy<br>Edit<br>SELECT NVL2(&#39;Data&#39;, &#39;Has Value&#39;, &#39;No Value&#39;) FROM DUAL;<br>-- Output: Has Value<br><br>SELECT NVL2(NULL, &#39;Has Value&#39;, &#39;No Value&#39;) FROM DUAL;<br>-- Output: No Value</pre><h3><strong>Question 4:</strong></h3><p>Give an output that contains just the first_name of the person having highest age.</p><p>a. Using max function</p><blockquote>select first_name from Customers where age = (select max(age) from Customers);</blockquote><p>b. without using max function</p><blockquote>select first_name from Customers order by age desc limit 1</blockquote><p>c. 3rd highest age Using window function</p><blockquote>select first_name from (select first_name, age,RANK( ) over (order by age Desc) as rnk from Customers ) ranked_Customers where rnk = 3;</blockquote><h3><strong>Question 5:</strong></h3><h3>Anatomy of SQL Query:</h3><p>SQL Execution Order (How SQL actually processes your query)</p><pre>SELECT department, COUNT(*) AS emp_count<br>FROM employees<br>WHERE salary &gt; 50000<br>GROUP BY department<br>HAVING COUNT(*) &gt; 3<br>ORDER BY emp_count DESC<br>LIMIT 5;</pre><pre><br><br>FROM – Identify the source table(s)<br><br>JOIN – Combine rows from related tables<br><br>WHERE – Filter rows before aggregation<br><br>GROUP BY – Group rows based on column(s)<br><br>HAVING – Filter groups after aggregation<br><br>SELECT – Pick columns or expressions to return<br><br>ORDER BY – Sort the result<br><br>LIMIT – Restrict number of rows returned</pre><h3>Question 6 :</h3><h3>Write a code to print reverse string:</h3><h3>Ist approach</h3><blockquote>s= hello</blockquote><blockquote>print(s[::-1])</blockquote><h3>2nd Approach</h3><blockquote>s= hello</blockquote><blockquote>reversed_str = “ ”</blockquote><blockquote>for char in s :</blockquote><blockquote>reversed_str = char + reversed_str</blockquote><blockquote>print(reversed_str)</blockquote><h3>Question 7:</h3><h3>Merge two dictionaries and what will happen to the common key?</h3><h3>dict1 = {‘a’: 1, ‘b’: 2}<br>dict2 = {‘b’: 3, ‘c’: 4}</h3><h3>Union method:</h3><blockquote>merged = dict1 | dict 2</blockquote><blockquote>print(merged)</blockquote><blockquote><strong>output</strong>: {‘a’: 1, ‘b’: 3, ‘c’: 4}</blockquote><h3>Loop-based merge (manual way)</h3><blockquote>merged = dict1.copy()</blockquote><blockquote>for key,value in dict2.items():</blockquote><blockquote>merged[key] = value</blockquote><blockquote>print(merged)</blockquote><h3><strong>Question 8:</strong></h3><h3>Given an list containing n distinct numbers taken from 0, 1, 2, …, n, find the one number that is missing from the list.<br>create a function check_missing() which will accept the list and returns missing number from list.<br>Assumptions:<br> 1. List can be of any length but only one number will be missing.<br> 2. List will always be sorted in ascending order.<br> 3. List can start from any number(it will not start from zero for every input)</h3><p>Input — [0, 1, 3]<br>Output — 2<br>Input — [3,4,5,7]<br>Output — 6</p><p><strong>Sum-Based Approach</strong></p><blockquote>def check_missing(nums):<br> start = nums[0]<br> end = nums[-1]<br> <br> # Expected sum from start to end<br> expected_sum = sum(range(start, end + 1))<br> <br> # Actual sum of the list<br> actual_sum = sum(nums)<br> <br> # Missing number is the difference<br> return expected_sum — actual_sum</blockquote><p><strong>Set-Based Approach</strong></p><blockquote>def check_missing(nums):<br> full_set = set(range(nums[0], nums[-1] + 1))<br> actual_set = set(nums)<br> missing_number = full_set — actual_set<br> return missing_number.pop()</blockquote><h4>difference between 9/2 9//2 and 9%2 in python</h4><blockquote>print(9 / 2) # Output: 4.5</blockquote><blockquote>print(9 // 2) # Output: 4</blockquote><blockquote>print(9 % 2) # Output: 1</blockquote><h3>Question 9:</h3><p><strong>Q: Given a dictionary where keys are file names and values are owner names, write a function that returns a new dictionary where each key is an owner and the value is a list of files owned by that person.</strong></p><p>✅ <strong>Using a normal dictionary</strong></p><pre>def group_files_by_owner(file_dict):<br>    result = {}<br>    for file, owner in file_dict.items():<br>        if owner not in result:<br>            result[owner] = []<br>        result[owner].append(file)<br>    return result</pre><h3>Explanation:</h3><pre>why a list here as we need dictinary ow these two line sare forming still not getting result[owner] = []<br>        result[owner].append(file)<br><br><br>ChatGPT said:<br>Great question! Let&#39;s break it down super simply to show why we use a list, and what exactly these two lines are doing:<br><br>🔁 Your Confusion:<br>Why do we use a list, and how do these two lines:<br><br>python<br>Copy<br>Edit<br>result[owner] = []<br>result[owner].append(file)<br>work together to form a dictionary?<br><br>🧠 Let&#39;s start with the goal:<br>We want to convert this:<br><br>python<br>Copy<br>Edit<br>{<br>    &#39;file1.txt&#39;: &#39;Alice&#39;,<br>    &#39;file2.txt&#39;: &#39;Bob&#39;,<br>    &#39;file3.txt&#39;: &#39;Alice&#39;<br>}<br>Into this:<br><br>python<br>Copy<br>Edit<br>{<br>    &#39;Alice&#39;: [&#39;file1.txt&#39;, &#39;file3.txt&#39;],<br>    &#39;Bob&#39;: [&#39;file2.txt&#39;]<br>}<br>So we want a dictionary where:<br><br>Key = Owner (like &#39;Alice&#39;)<br><br>Value = List of files they own (like [&#39;file1.txt&#39;, &#39;file3.txt&#39;])<br><br>🔍 What is result[owner] = [] doing?<br>This line is just creating an empty list for that owner if it doesn&#39;t already exist.<br><br>Example:<br><br>python<br>Copy<br>Edit<br>if owner not in result:<br>    result[owner] = []<br>It says:<br><br>&quot;Hey, if Alice isn&#39;t in the dictionary yet, make a new entry for her and set it to an empty list.&quot;<br><br>At this point, the dictionary looks like:<br><br>python<br>Copy<br>Edit<br>{<br>    &#39;Alice&#39;: []<br>}<br>🔍 What is result[owner].append(file) doing?<br>This line says:<br><br>&quot;Now take that list (which is currently empty), and add the current file name to it.&quot;<br><br>Example:<br><br>python<br>Copy<br>Edit<br>result[&#39;Alice&#39;].append(&#39;file1.txt&#39;)<br>Now the dictionary becomes:<br><br>python<br>Copy<br>Edit<br>{<br>    &#39;Alice&#39;: [&#39;file1.txt&#39;]<br>}<br>Later if we find &#39;file3.txt&#39; is also owned by Alice, this line adds that too:<br><br>python<br>Copy<br>Edit<br>result[&#39;Alice&#39;].append(&#39;file3.txt&#39;)<br>Final dictionary:<br><br>python<br>Copy<br>Edit<br>{<br>    &#39;Alice&#39;: [&#39;file1.txt&#39;, &#39;file3.txt&#39;]<br>}</pre><h3>Pyspark Code:</h3><pre>from pyspark.sql import SparkSession<br>from pyspark.sql.functions import collect_list<br><br># Start Spark session<br>spark = SparkSession.builder.appName(&quot;GroupFilesByOwner&quot;).getOrCreate()<br><br># Input data<br>data = [<br>    (&quot;file1.txt&quot;, &quot;Alice&quot;),<br>    (&quot;file2.txt&quot;, &quot;Bob&quot;),<br>    (&quot;file3.txt&quot;, &quot;Alice&quot;),<br>    (&quot;file4.txt&quot;, &quot;Charlie&quot;),<br>    (&quot;file5.txt&quot;, &quot;Bob&quot;)<br>]<br><br># Create DataFrame<br>df = spark.createDataFrame(data, [&quot;file_name&quot;, &quot;owner&quot;])<br><br># Group by owner and collect file names into a list<br>df.groupBy(&quot;owner&quot;).agg(collect_list(&quot;file_name&quot;).alias(&quot;files&quot;))<br><br># Show result<br>grouped_df.show(truncate=False)</pre><h3>question 10 :</h3><h3>Snowflake Architecture:</h3><pre>❄️ Snowflake Architecture Overview<br>Snowflake uses a cloud-native architecture that separates compute, storage, and services, making it highly scalable, flexible, and cost-efficient.<br><br>📐 Three-Layered Architecture<br>markdown<br>Copy<br>Edit<br>┌──────────────────────┐<br>│   Cloud Services     │   ← Control &amp; Management<br>└──────────────────────┘<br>          ▲<br>          │<br>┌──────────────────────┐<br>│   Compute Layer      │   ← Virtual Warehouses<br>└──────────────────────┘<br>          ▲<br>          │<br>┌──────────────────────┐<br>│   Storage Layer      │   ← Persistent Data<br>└──────────────────────┘<br>🔹 1. Storage Layer<br>Stores structured &amp; semi-structured data (JSON, Avro, Parquet, etc.)<br><br>Automatically compressed &amp; encrypted.<br><br>Managed by Snowflake (no manual tuning or indexing).<br><br>Scales independently of compute.<br><br>✅ Think of it like your data warehouse &quot;hard disk&quot;.<br><br>🔹 2. Compute Layer<br>Made up of Virtual Warehouses.<br><br>Each warehouse is an independent compute cluster.<br><br>Responsible for query processing.<br><br>Multiple warehouses can run in parallel without affecting each other.<br><br>✅ You can scale up/down depending on workload.<br><br>🔹 3. Cloud Services Layer<br>Handles:<br><br>Authentication &amp; security<br><br>Query parsing &amp; optimization<br><br>Metadata management<br><br>Infrastructure management<br><br>✅ Acts like the brain of Snowflake – managing everything automatically.</pre><h3>Question 11:</h3><h3>What Do ETL and ELT Stand For?</h3><pre>🔁 <br><br>ETL ELT<br>Extract → Transform → Load Extract → Load → Transform<br>⚙️ Meaning:<br>🔷 ETL:<br>Extract data from source systems<br><br>Transform it (clean, join, filter, etc.) on a middle-tier server or ETL tool<br><br>Load the transformed data into the data warehouse<br><br>✅ Used when:<br><br>Data warehouse can&#39;t handle heavy transformations<br><br>Pre-processing is needed before storing<br><br>🔶 ELT:<br>Extract data from source systems<br><br>Load raw data into the data warehouse or data lake<br><br>Transform the data inside the warehouse using SQL or processing engines<br><br>✅ Used when:<br><br>Using modern cloud data warehouses like Snowflake, BigQuery, Redshift<br><br>You want to store raw data for auditing or future use<br><br>🔍 Key Differences<br><br>Feature ETL ELT<br>Transformation Before loading After loading<br>Storage Stores only processed data Can store both raw and processed<br>Speed Slower (depends on ETL tool/server) Faster with modern cloud platforms<br>Tooling Informatica, Talend, SSIS, etc. Spark SQL, dbt, native SQL<br>Flexibility Less flexible for schema change More flexible (raw data preserved)<br>Best For Traditional on-prem DWH Cloud-based modern DWH</pre><p>Question 12:</p><h3>1️⃣ OFFSET in SQL</h3><pre>1️⃣ OFFSET in SQL<br>📌 What is OFFSET?<br>OFFSET is used to skip a certain number of rows in the result set — usually used with LIMIT.<br><br>✅ Syntax:<br>sql<br>Copy<br>Edit<br>SELECT * FROM table_name<br>ORDER BY column_name<br>LIMIT 5 OFFSET 10;</pre><h3>📌 What is ILIKE?</h3><pre>2️⃣ ILIKE in SQL<br>📌 What is ILIKE?<br>ILIKE is a case-insensitive version of LIKE.<br><br>✅ Syntax:<br>sql<br>Copy<br>Edit<br>SELECT * FROM users<br>WHERE name ILIKE &#39;john%&#39;;</pre><h3>— — — — — — — AMEX INTERVIEW — — — — — — — — -</h3><h3>QUESTION 1 :</h3><p><strong>Given Table A and Table B with ID columns, write SQL queries to perform A minus B (A — B) and a LEFT JOIN from A to B on ID.You are given two tables, Table A (IDs: 1, 2, 3, 3, 4, 6) and Table B (IDs: 1, 2, 2, 3, 5, 5, 6).</strong></p><ol><li><strong>Using </strong><strong>NOT IN for A - B (Set Difference):</strong></li></ol><blockquote>select id from A where id not in (select id from B)</blockquote><p>2. <strong>Using </strong><strong>LEFT JOIN for A - B</strong></p><blockquote>select a.id from a left join b on a.id = b.id where b.id is null</blockquote><h3>Question 2:</h3><p><strong>Remove the duplicates where product_id and name is same via sql PRODUCT_ID Name Price</strong></p><p><strong>1 Laptop 1000</strong></p><p><strong>2 Tablet 600</strong></p><p><strong>1 Laptop 1000</strong></p><p><strong>4 Phone 800</strong></p><h3>1. Using DISTINCT:</h3><blockquote>select Distinct Product_id , name, Price from Product</blockquote><h3>2. Using Row_Number:</h3><p>with Ranked_product as(</p><p>select Product_id, name , product_id , row_number() over (partition by product_id, name order by name desc) as row_num from Product</p><p>)</p><p>select Product_id, name , product_id from Ranked_product where row_num =1</p><h3>Question 3:</h3><h3>Every day you are getting the data from different sources and we are updating the dataset so over a period of time the performance starts degrading, can you tell me what are the different ways we can that in spark?</h3><p>Before this read this: <a href="https://medium.com/@ankitjuneja/spark-performance-optimizations-46204d99516f">https://medium.com/@ankitjuneja/spark-performance-optimizations-46204d99516f</a></p><p>In Apache Spark, when the performance of a job or a pipeline starts degrading due to changing or growing datasets, there are several strategies you can apply to manage and improve performance. This situation often arises because Spark applications are dynamic, and new data sources or larger datasets require different optimization techniques. Below are the different ways to address performance degradation over time in Spark:</p><h3>1. Efficient Data Partitioning</h3><ul><li><strong>Repartitioning</strong>: If the data volume increases over time, the default number of partitions might not be sufficient. Repartitioning the data (using repartition() or coalesce()) can help spread the workload evenly across executors.</li><li><strong>Partitioning by Columns</strong>: When reading from data sources like HDFS, Hive, or Parquet, ensure that the data is partitioned by meaningful columns (e.g., date, region). This will allow Spark to read only the relevant data and skip unnecessary partitions (partition pruning).</li><li><strong>Dynamic Partitioning</strong>: Over time, if new partitions are added to your dataset, Spark can automatically detect and prune the unused partitions for efficiency.</li></ul><h3>2. Caching and Persistence</h3><ul><li><strong>Cache Intermediate Results</strong>: When data is reused multiple times in an application, caching it in memory (using cache() or persist()) can improve performance. However, you should periodically clean the cache if the data grows to avoid memory overload.</li><li><strong>Storage Level Tuning</strong>: Choose appropriate storage levels (MEMORY_ONLY, MEMORY_AND_DISK, DISK_ONLY) based on the size of your data. Larger data that doesn’t fit in memory can benefit from MEMORY_AND_DISK persistence.</li></ul><h3>3. Data Skew Mitigation</h3><ul><li><strong>Salting</strong>: Data skew occurs when certain partitions have significantly more data than others. This can cause performance degradation as some tasks take longer to complete. One solution is to introduce “salting,” which spreads data more evenly across partitions by adding a random key.</li><li><strong>Broadcast Joins</strong>: For skewed joins, you can use Spark’s broadcast join feature (broadcast()) when one side of the join is small enough to fit into memory. This prevents large shuffle operations.</li></ul><h3>4. Efficient File Formats</h3><ul><li><strong>Columnar Formats (Parquet, ORC)</strong>: Ensure that you’re using efficient file formats like Parquet or ORC, which store data in a compressed columnar format. These formats support predicate pushdown and compression, reducing both IO and memory usage.</li><li><strong>Compression</strong>: Use appropriate compression formats like Snappy or Zlib to reduce the amount of data read from storage, which improves performance when working with large datasets.</li></ul><h3>5. Optimizing Shuffles and Joins</h3><ul><li><strong>Reduce Shuffling</strong>: Shuffling is one of the most expensive operations in Spark. You can reduce shuffles by avoiding wide transformations (like groupByKey, reduceByKey) when possible. Using narrow transformations (like map, filter) or minimizing group operations can improve performance.</li><li><strong>Skewed Joins</strong>: For joins with skewed data, consider using skew hints or the skewJoinOptimization feature in Spark, which can dynamically adjust the join strategy to avoid overloading certain partitions.</li><li><strong>Broadcast Joins</strong>: For smaller tables, use broadcast joins to minimize shuffle data (broadcast()).</li></ul><h3>6. Adaptive Query Execution (AQE)</h3><ul><li>Spark 3.0+ introduced <strong>Adaptive Query Execution (AQE)</strong>, which dynamically optimizes the query plan at runtime based on the actual data being processed. AQE can:</li><li>Dynamically adjust the number of partitions based on the size of the shuffle data.</li><li>Automatically use better join strategies (like converting sort-merge joins to broadcast joins).</li><li>Automatically handle skewed data in joins.</li><li>Enabling AQE can significantly improve performance as the system adjusts to changing data characteristics.</li></ul><h3>7. Optimize Resource Allocation</h3><ul><li><strong>Executor and Driver Memory</strong>: Ensure that the executors and driver have sufficient memory and cores allocated. Increase the memory or core allocation as the data size grows.</li><li><strong>Dynamic Allocation</strong>: Enable dynamic allocation (spark.dynamicAllocation.enabled) to let Spark automatically adjust the number of executors based on the workload. This prevents resource contention as the data grows over time.</li></ul><h3>8. Optimize Query Plan</h3><ul><li><strong>Predicate Pushdown</strong>: When working with external data sources like Parquet, Hive, or JDBC, make sure that predicates (filters) are pushed down to the data source level. This minimizes the amount of data that Spark needs to read.</li><li><strong>Project Pushdown</strong>: Only select the columns that you need to process (using .select()) rather than loading the entire dataset into memory. This is especially helpful when working with large tables or datasets.</li></ul><h3>9. Monitor and Tune Garbage Collection (GC)</h3><ul><li>Over time, as the dataset size grows, you might encounter more frequent garbage collection pauses, especially with large datasets. You can optimize GC by:</li><li>Tuning the size of the memory heap.</li><li>Using the G1GC garbage collector, which is optimized for large datasets (-XX:+UseG1GC).</li><li>Monitoring GC overhead in the Spark UI and adjusting the executor memory accordingly.</li></ul><h3>10. Dataframe API vs. RDD API</h3><ul><li><strong>DataFrame and Dataset API</strong>: Use DataFrames and Datasets instead of RDDs whenever possible. The DataFrame API is more optimized, allowing Spark’s Catalyst optimizer to generate efficient query execution plans, including optimizations like predicate pushdown and vectorized operations.</li></ul><h3>11. Incremental Data Processing</h3><ul><li><strong>Structured Streaming</strong>: For applications that handle continuously growing data (e.g., daily data from different sources), consider using <strong>Structured Streaming</strong>. It allows you to process data incrementally and perform operations like aggregations and joins in a streaming fashion, reducing the overall processing overhead compared to batch processing.</li></ul><h3>12. Z-Ordering and Data Clustering</h3><ul><li><strong>Z-Ordering</strong>: For large datasets, especially in Delta Lake, Z-ordering optimizes how data is stored on disk, making queries on frequently filtered columns (like dates or geographies) faster by reducing the amount of data read from disk.</li><li><strong>Data Clustering</strong>: Grouping related data together on disk helps reduce the amount of data Spark needs to read for queries. This is particularly effective for queries that frequently filter or join on the same columns.</li></ul><h3>13. Handling Growing Metadata in Hive Metastore</h3><ul><li>As datasets grow, the metadata stored in the Hive metastore can become large, which can slow down operations. Using <strong>Partition Discovery</strong> and <strong>Table Pruning</strong> can help avoid querying excessive metadata. You can also periodically compact your Hive partitions to reduce metadata.</li></ul><h3>14. Compacting Small Files</h3><ul><li>Over time, small files can accumulate, leading to inefficient reading due to many I/O operations. You can periodically compact small files into larger files (especially with Parquet/ORC) to improve read performance and reduce the overhead of managing many small files.</li></ul><h3>15. DataFrame and Dataset API Optimizations</h3><ul><li><strong>Avoid UDFs when Possible</strong>: User Defined Functions (UDFs) can hurt performance, as they operate on individual rows and bypass Spark’s Catalyst optimizer. Use native DataFrame functions or Spark SQL expressions whenever possible for better performance.</li><li><strong>SQL Query Tuning</strong>: Use EXPLAIN plans to analyze and tune the Spark SQL queries. Ensure that the query plans are optimized for your growing data volumes.</li></ul><p>By combining these techniques, you can improve Spark’s performance and ensure that your application remains scalable as the dataset grows over time.</p><h3>Question 4:</h3><h3>What are the different ways of loading the data in spark?</h3><p>first read this:</p><p><a href="https://medium.com/@ankitjuneja/pyspark10-sparksession-da327e3d56bd">https://medium.com/@ankitjuneja/pyspark10-sparksession-da327e3d56bd</a></p><p>spark =. SparkSession.builder.appName(“i am confident”).getOrCreate()</p><p>spark.read.csv.option(“header”,true).option(“inferSchema”, true).option(path, “/asd”).load</p><p>spark.write.csv.option(“asd”).partitionBy(“asd”).BucketBy(“asd”,3).mode(“Overwrite”).saveAsTable</p><pre>1. Loading Data from HDFS, Local File System, or S3<br>Spark can read data from various file systems like HDFS, local file systems, and cloud storage like Amazon S3, Google Cloud Storage, or Azure Blob Storage.<br><br>Example (CSV, Parquet, ORC, JSON):<br>python<br>Copy<br>Edit<br># Load CSV<br>df_csv = spark.read.format(&quot;csv&quot;).option(&quot;header&quot;, &quot;true&quot;).load(&quot;/path/to/data.csv&quot;)<br><br># Load Parquet<br>df_parquet = spark.read.format(&quot;parquet&quot;).load(&quot;/path/to/data.parquet&quot;)<br><br># Load ORC<br>df_orc = spark.read.format(&quot;orc&quot;).load(&quot;/path/to/data.orc&quot;)<br><br># Load JSON<br>df_json = spark.read.format(&quot;json&quot;).load(&quot;/path/to/data.json&quot;)<br>Options:<br>.option(&quot;header&quot;, &quot;true&quot;): For files like CSV, you can specify that the first row is the header.<br><br>.option(&quot;inferSchema&quot;, &quot;true&quot;): To infer the schema automatically from the data.<br><br>S3 Example:<br>python<br>Copy<br>Edit<br>df_s3 = spark.read.format(&quot;parquet&quot;).load(&quot;s3a://my-bucket/path/to/data.parquet&quot;)<br>2. Loading Data from Relational Databases (JDBC)<br>Spark supports loading data from relational databases using JDBC, allowing connections to databases like MySQL, PostgreSQL, Oracle, SQL Server, etc.<br><br>Example:<br>python<br>Copy<br>Edit<br>jdbc_url = &quot;jdbc:mysql://hostname:3306/database&quot;<br>connection_properties = {<br>    &quot;user&quot;: &quot;username&quot;,<br>    &quot;password&quot;: &quot;password&quot;,<br>    &quot;driver&quot;: &quot;com.mysql.cj.jdbc.Driver&quot;<br>}<br><br>df_jdbc = spark.read.jdbc(url=jdbc_url, table=&quot;table_name&quot;, properties=connection_properties)<br>Options:<br>.option(&quot;dbtable&quot;, &quot;table_name&quot;): To specify the table.<br><br>.option(&quot;query&quot;, &quot;SELECT * FROM table WHERE ...&quot;): To use SQL queries for more customized data loading.<br><br>Partitioning:<br>You can improve performance for large datasets by partitioning the data across the Spark cluster:<br><br>python<br>Copy<br>Edit<br>df_jdbc_partitioned = spark.read.jdbc(<br>    url=jdbc_url,<br>    table=&quot;table_name&quot;,<br>    properties=connection_properties,<br>    column=&quot;id&quot;,<br>    lowerBound=1,<br>    upperBound=10000,<br>    numPartitions=10<br>)<br>3. Loading Data from Apache Hive<br>If you&#39;re using a Hive metastore, Spark can directly query and load data from Hive tables. Hive tables can be managed or external, and the data can be in formats like Parquet, ORC, CSV, etc.<br><br>Example:<br>python<br>Copy<br>Edit<br>spark.sql(&quot;USE my_database&quot;)  # Set the database<br>df_hive = spark.sql(&quot;SELECT * FROM my_table&quot;)<br>Alternatively, you can use the table method:<br><br>python<br>Copy<br>Edit<br>df_hive = spark.read.table(&quot;my_database.my_table&quot;)<br>4. Loading Data from Apache Kafka<br>Spark Structured Streaming has built-in support for loading data from Kafka, making it easy to integrate Spark with Kafka for real-time processing.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_kafka = spark.readStream.format(&quot;kafka&quot;) \<br>    .option(&quot;kafka.bootstrap.servers&quot;, &quot;localhost:9092&quot;) \<br>    .option(&quot;subscribe&quot;, &quot;topic_name&quot;) \<br>    .load()<br><br>df_kafka.selectExpr(&quot;CAST(key AS STRING)&quot;, &quot;CAST(value AS STRING)&quot;)<br>5. Loading Data from Apache Cassandra<br>Spark has a connector for Cassandra that allows you to load data from Cassandra tables.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_cassandra = spark.read \<br>    .format(&quot;org.apache.spark.sql.cassandra&quot;) \<br>    .options(table=&quot;table_name&quot;, keyspace=&quot;keyspace_name&quot;) \<br>    .load()<br>6. Loading Data from HBase<br>Using the HBase-Spark connector, Spark can interact with HBase to load data from HBase tables.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_hbase = spark.read \<br>    .format(&quot;org.apache.hadoop.hbase.spark&quot;) \<br>    .option(&quot;hbase.table&quot;, &quot;table_name&quot;) \<br>    .load()<br>7. Loading Data from Delta Lake<br>Delta Lake is an ACID-compliant storage layer for data lakes, and Spark can load data from Delta tables.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_delta = spark.read.format(&quot;delta&quot;).load(&quot;/path/to/delta/table&quot;)<br>8. Loading Data from Avro Format<br>Spark supports loading data in Avro format using the avro package.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_avro = spark.read.format(&quot;avro&quot;).load(&quot;/path/to/data.avro&quot;)<br>9. Loading Data from NoSQL Databases (MongoDB)<br>Using the MongoDB Connector for Apache Spark, you can load data from MongoDB collections.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_mongo = spark.read \<br>    .format(&quot;com.mongodb.spark.sql.DefaultSource&quot;) \<br>    .option(&quot;uri&quot;, &quot;mongodb://localhost:27017/database.collection&quot;) \<br>    .load()<br>10. Loading Data from ElasticSearch<br>The Elasticsearch-Spark connector allows you to load data from Elasticsearch indexes.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_es = spark.read \<br>    .format(&quot;org.elasticsearch.spark.sql&quot;) \<br>    .option(&quot;es.nodes&quot;, &quot;localhost&quot;) \<br>    .option(&quot;es.port&quot;, &quot;9200&quot;) \<br>    .load(&quot;index_name/type_name&quot;)<br>11. Loading Data from Google BigQuery<br>With the BigQuery connector, you can load data from BigQuery tables into Spark.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_bq = spark.read \<br>    .format(&quot;bigquery&quot;) \<br>    .option(&quot;project&quot;, &quot;my_project&quot;) \<br>    .option(&quot;table&quot;, &quot;my_dataset.my_table&quot;) \<br>    .load()<br>12. Loading Data from CSV Files<br>Spark provides a direct method to load CSV files using the .csv() method.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_csv = spark.read.option(&quot;header&quot;, &quot;true&quot;).csv(&quot;/path/to/data.csv&quot;)<br>13. Loading Data from JSON Files<br>Spark can load JSON files natively.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_json = spark.read.json(&quot;/path/to/data.json&quot;)<br>14. Loading Data from External APIs<br>If you need to load data from external APIs (e.g., REST APIs), you can use libraries like requests in Python or java.net.HttpURLConnection in Java/Scala to fetch the data, then convert it into an RDD or DataFrame.<br><br>Example (Python):<br>python<br>Copy<br>Edit<br>import requests<br>data = requests.get(&quot;https://api.example.com/data&quot;).json()<br>df_api = spark.createDataFrame(data)<br>15. Loading Data using Custom Data Sources<br>Spark allows you to implement custom data sources by extending its DataSourceV2 API. You can create connectors for any source (e.g., proprietary systems) and load the data into Spark.<br><br>Example:<br>python<br>Copy<br>Edit<br>df_custom = spark.read.format(&quot;com.example.customsource&quot;).load()<br>16. Loading Data from Memory (In-Memory Collections)<br>You can create Spark DataFrames directly from Python/Scala lists or RDDs for testing or quick prototyping.<br><br>Example (Python List):<br>python<br>Copy<br>Edit<br>data = [(&quot;Alice&quot;, 34), (&quot;Bob&quot;, 45), (&quot;Charlie&quot;, 23)]<br>df_memory = spark.createDataFrame(data, [&quot;Name&quot;, &quot;Age&quot;])</pre><h3>Questions 5 :</h3><h3><em>Optimize the following SQL query and rewrite it in PySpark, applying relevant optimization techniques. The </em><em>customers table is in TB, </em><em>orders in GB, and </em><em>products in MB:</em></h3><pre>SELECT c.name, SUM(o.amount) AS total_amount<br>FROM customers c<br>JOIN orders o ON c.customer_id = o.customer_id<br>JOIN products p ON o.product_id = p.product_id<br>WHERE o.order_date BETWEEN &#39;2023-01-01&#39; AND &#39;2023-12-31&#39;<br>GROUP BY c.name<br>ORDER BY total_amount DESC;</pre><h3>Optimization Techniques</h3><pre><br>Avoiding Cartesian Products in Joins: <br>The query is performing multiple joins. <br>Ensuring that the join keys (customer_id, product_id) are indexed or <br>partitioned will help avoid cartesian products or large shuffle operations.<br><br>Predicate Pushdown: <br>Since the query filters orders based on a date range (WHERE o.order_date BETWEEN &#39;2023-01-01&#39; AND &#39;2023-12-31&#39;), it&#39;s important that the filter is applied as early as possible. This can be done via predicate pushdown in Spark to minimize the data read.<br><br>Broadcast Joins: The products table is relatively small (MB scale) compared to customers and orders. We can leverage broadcast joins to load the smaller table (products) into memory on each worker, reducing the need for shuffling and improving join performance.<br><br>Partitioning: Given the large size of the customers and orders tables, you should partition them effectively. Partitioning orders by order_date and customers by customer_id would improve query performance by reducing the amount of data read.<br><br>Use of Catalyst Optimizer: Spark&#39;s Catalyst optimizer will handle many optimization tasks automatically, but ensuring that DataFrame transformations are written in a way that avoids unnecessary actions and shuffles is critical.<br><br>Caching Intermediate Results: If the data is being reused in multiple operations or steps, caching intermediate results (e.g., filtered orders) can help improve performance, especially when working with large datasets.<br><br>Column Pruning: Select only the necessary columns in the DataFrame to reduce the amount of data transferred between the stages. In this case, we only need specific columns like customer_id, order_date, and amount from the orders table, name from the customers table, and product_id from the products table.<br><br>Data Skew Management: If there&#39;s data skew (e.g., some customers or products having significantly more orders), introducing techniques like salting or repartitioning can help.</pre><h3>Code:</h3><p>from pyspark import SparkSession</p><pre>from pyspark.sql import SparkSession<br>from pyspark.sql.functions import col, sum as _sum, desc<br>from pyspark.sql import functions as F<br><br># Initialize Spark Session<br>spark = SparkSession.builder \<br>    .appName(&quot;Optimize SQL Query&quot;) \<br>    .getOrCreate()<br><br># Load the tables<br># Assuming the data is stored in Parquet format or any other efficient columnar format<br>customers_df = spark.read.format(&quot;parquet&quot;).load(&quot;/path/to/customers&quot;)<br>orders_df = spark.read.format(&quot;parquet&quot;).load(&quot;/path/to/orders&quot;)<br>products_df = spark.read.format(&quot;parquet&quot;).load(&quot;/path/to/products&quot;)<br><br># 1. Apply partitioning strategy<br># Assuming customers are partitioned by customer_id, orders by order_date<br># This can be done during table creation or by re-partitioning the DataFrame<br><br># 2. Filter the &#39;orders&#39; table early (Predicate Pushdown)<br># Filter the orders within the specified date range<br>filtered_orders_df = orders_df.filter((col(&quot;order_date&quot;) &gt;= &quot;2023-01-01&quot;) &amp; <br>                                      (col(&quot;order_date&quot;) &lt;= &quot;2023-12-31&quot;))<br><br># 3. Broadcast smaller &#39;products&#39; table to avoid shuffling (Broadcast Join)<br>broadcast_products_df = F.broadcast(products_df)<br><br># 4. Perform the joins efficiently<br>joined_df = customers_df \<br>    .join(filtered_orders_df, customers_df.customer_id == filtered_orders_df.customer_id) \<br>    .join(broadcast_products_df, filtered_orders_df.product_id == broadcast_products_df.product_id)<br><br># 5. Select only the necessary columns<br>selected_df = joined_df.select(customers_df.name, filtered_orders_df.amount)<br><br># 6. Group by &#39;name&#39; and calculate the total amount for each customer<br>grouped_df = selected_df<br>.groupBy(&quot;name&quot;)<br>.agg(_sum(&quot;amount&quot;)<br>.alias(&quot;total_amount&quot;))<br><br># 7. Order the results by &#39;total_amount&#39; in descending order<br>result_df = grouped_df.orderBy(desc(&quot;total_amount&quot;))<br><br># 8. (Optional) Cache the final result if it&#39;s reused elsewhere<br>result_df.cache()<br><br># Show the final result<br>result_df.show()</pre><h3>Question 6:</h3><h3>You are working with a DataFrame transaction_df that appears highly skewed on customer_id based on UI insights. How would you confirm this data skew programmatically?</h3><p><strong>Count the Frequency of Each customer_id:</strong><br>The simplest way to check for skew is to group by customer_id and count the number of transactions for each customer. Then, we can sort the results to see the distribution and check if a few customer_id values dominate the dataset.</p><pre><br><br>from pyspark.sql.functions import col, count<br><br># Group by &#39;customer_id&#39; and count the number of records for each customer<br>customer_count_df = transaction_df.groupBy(&quot;customer_id&quot;).agg(count(&quot;*&quot;).alias(&quot;transaction_count&quot;))<br><br># Order by transaction count in descending order to see the distribution<br>customer_count_df.orderBy(col(&quot;transaction_count&quot;).desc()).show(10)</pre><h3>Question 7:</h3><h3>write a code for salting Technique?</h3><pre>import random<br>from pyspark.sql.functions import col, count, lit, concat<br><br># Step 1: Define the number of salt values (based on how much skew you expect)<br>num_salts = 10  # Adjust based on your data size and skew<br><br># Step 2: Add a salt column<br># Create a new column &#39;salt&#39; with a random value between 0 and num_salts-1<br>salted_df = df.withColumn(&quot;salt&quot;, (col(&quot;customer_id&quot;) % num_salts))  # Deterministic salting<br><br># Step 3: Combine &#39;customer_id&#39; and &#39;salt&#39; to create a new salted key<br># We use concat to combine &#39;customer_id&#39; and &#39;salt&#39;<br>salted_df = salted_df.withColumn(&quot;salted_customer_id&quot;, concat(col(&quot;customer_id&quot;), lit(&quot;_&quot;), col(&quot;salt&quot;)))<br><br># Step 4: Perform the grouping and aggregation using the salted key<br># Group by the new &#39;salted_customer_id&#39; to distribute the skew<br>grouped_salted_df = salted_df.groupBy(&quot;salted_customer_id&quot;).agg(count(&quot;*&quot;).alias(&quot;transaction_count&quot;))<br><br># Step 5: Remove the salt by re-grouping on the original &#39;customer_id&#39;<br># We need to aggregate the results back to the original customer_id<br>final_df = grouped_salted_df.withColumn(&quot;customer_id&quot;, col(&quot;salted_customer_id&quot;).substr(1, length(col(&quot;customer_id&quot;)))) \<br>                            .groupBy(&quot;customer_id&quot;).agg(sum(&quot;transaction_count&quot;).alias(&quot;total_transactions&quot;))<br><br># Step 6: Show the final result<br>final_df.show()</pre><h3>#python interview questions.</h3><h3>Question 1:</h3><h3>you have [1,2,3,4,5,6] list , how to do you swap 2nd item to 2nd last element in python. input=[1,2,3,4,5,6] output=[1,5,3,4,2,6]</h3><h3>Ist approach:</h3><pre>input=[1,2,3,4,5,6]</pre><pre>temp = input[1]</pre><pre>input[1] = input[-2]</pre><pre>input[-2] =  temp<br></pre><pre>print(input)</pre><h3>2nd approach:</h3><pre>input_list = [1, 2, 3, 4, 5, 6]<br>input_list[1], input_list[-2] = input_list[-2], input_list[1]<br>print(input_list)</pre><pre>✅ Step-by-Step Execution:<br>Original List:</pre><pre>python<br>Copy<br>Edit<br>input_list = [1, 2, 3, 4, 5, 6]<br>                ↑           ↑<br>              index 1     index -2 (which is 4)<br>Right-hand Side (RHS) is evaluated first:</pre><pre>python<br>Copy<br>Edit<br>(input_list[-2], input_list[1]) → (5, 2)<br>Tuple unpacking happens on the left:</pre><pre>python<br>Copy<br>Edit<br>input_list[1], input_list[-2] = 5, 2<br>So now:</pre><pre>input_list[1] becomes 5</pre><pre>input_list[-2] (which is input_list[4]) becomes 2</pre><pre>✅ Final List:<br>python<br>Copy<br>Edit<br>[1, 5, 3, 4, 2, 6]<br>🔥 Why This Works:<br>The assignment happens simultaneously: Python creates a temporary tuple (5, 2) and unpacks it back to the variables in one step.</pre><pre>So there&#39;s no need for a temp variable, and there&#39;s no risk of overwriting one value before storing the other.</pre><h3>Question 2:</h3><h3>#write a program to reverse your name.</h3><h3>Ist approach:</h3><pre>name = &quot;ankit&quot;</pre><pre>print(name[::-1])</pre><h3>2nd Approach:</h3><h3>For length it is genrally -1 and for range it is generaly +1</h3><pre>name = &quot;ankit&quot;<br>reversed_string = &quot;&quot;<br>for i in range(len(name) - 1, -1, -1):<br>    reversed_string = reversed_string + name[i]<br>print(reversed_string)<br></pre><pre>#tikna</pre><h3>Now change the order of 4th line:</h3><pre>name = &quot;ankit&quot;<br>reversed_string = &quot;&quot;<br>for i in range(len(name) - 1, -1, -1):<br>    reversed_string = name[i] + reversed_string<br>print(reversed_string)</pre><pre>#ankit</pre><h3>Approach 3 :</h3><h3>for changing the reverse orders for words and not for characters:</h3><pre>name = &quot;ankit is a gem&quot;</pre><pre>name2 = name.split()</pre><pre>name3 = &quot; &quot;.join(name2[::-1])</pre><pre>print(name3)</pre><pre>#gem a is ankit</pre><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*0-GFJpQLaedwmckbEAXNkw@2x.jpeg" /></figure><p>for loop chalana padega</p><pre>text = &quot;i love coding&quot;<br>words = text.split()<br>reversed_words = [word[::-1] for word in words]<br>reversed_text = &quot; &quot;.join(reversed_words)<br>print(reversed_text)</pre><p><strong>Question 3:</strong></p><p><strong>How to open a file in python and how to execute a sql query in python write a simple program?</strong></p><h3>✅ 1. How to Open a File in Python</h3><p>You can open and read a file using the built-in open() function.</p><pre># Open and read a text file<br>with open(&#39;data.txt&#39;, &#39;r&#39;) as file:<br>    content = file.read()</pre><h4>✅ 2. How to Execute a SQL Query in Python</h4><p>To run SQL in Python, we usually use the sqlite3 module (for SQLite), or a package like pyodbc, psycopg2 (PostgreSQL), or sqlalchemy.</p><p>Here’s a <strong>simple example</strong> using sqlite3:</p><pre># Create a connection to a SQLite database (or memory DB)<br>conn = sqlite3.connect(&#39;my_database.db&#39;)  # Use &#39;:memory:&#39; for in-memory DB</pre><pre># Create a cursor to execute SQL<br>cursor = conn.cursor()</pre><pre># Create a sample table<br>cursor.execute(&#39;&#39;&#39;<br>    CREATE TABLE IF NOT EXISTS Employees (<br>        ID INTEGER PRIMARY KEY,<br>        Name TEXT,<br>        Department TEXT<br>    )<br>&#39;&#39;&#39;)</pre><pre># Insert sample data<br>cursor.execute(&quot;INSERT INTO Employees (Name, Department) VALUES (&#39;Alice&#39;, &#39;HR&#39;)&quot;)<br>cursor.execute(&quot;INSERT INTO Employees (Name, Department) VALUES (&#39;Bob&#39;, &#39;IT&#39;)&quot;)</pre><pre># Commit the changes<br>conn.commit()</pre><pre># Run a SELECT query<br>cursor.execute(&quot;SELECT * FROM Employees&quot;)<br>rows = cursor.fetchall()</pre><pre># Print the results<br>for row in rows:<br>    print(row)</pre><pre># Close the connection<br>conn.close()</pre><h3>Question 4:</h3><h3>a= {1,2,3,6,7,8,9,4} b = {2}</h3><h3>How do we know b is a subset of a in python write a simple code?</h3><pre>a = {1, 2, 3, 6, 7, 8, 9, 4}<br>b = {2}<br><br>is_subset = True  # Assume b is a subset until proven otherwise<br><br>for item in b:<br>    if item not in a:<br>        is_subset = False<br>        break  # No need to check further<br><br>if is_subset:<br>    print(&quot;b is a subset of a (checked using loop)&quot;)<br>else:<br>    print(&quot;b is NOT a subset of a (checked using loop)&quot;)</pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=92c5726b84c4" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[Data Engineer Questions backup -6:]]></title>
            <link>https://medium.com/@ankitjuneja/data-engineer-questions-268464dc0aed?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/268464dc0aed</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Sat, 19 Oct 2024 14:31:22 GMT</pubDate>
            <atom:updated>2025-08-22T04:56:01.009Z</atom:updated>
            <content:encoded><![CDATA[<h3>Priceplan code</h3><pre>Pricelan_code:<br>priceplan_test<br><br><br>Pricelan_code:<br>entity<br>exceptions<br>helper<br>job<br>udfs<br>util<br><br><br><br><br><br><br>hepler<br><br>priceplanhelper<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.helper<br><br>.marriott.mgpapp.spark.priceplan.ingestion.util.BuilderUtil.prepareDocumentDBReaderWithFilter<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{Deserializer, InputType, JobType, Serializer}<br>import com.marriott.rm.platform.util.{ConsumerHelperBuilder, ProducerHelperBuilder, ReaderBuilder}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql._<br>import org.apache.spark.sql.functions._<br>import org.apache.spark.sql.types._<br><br>import java.time.format.DateTimeFormatter<br>import java.time.{LocalDate, LocalDateTime}<br><br>trait PricePlanIngestionHelper extends Serializable with StreamingJob {<br><br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br>  val customMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, loggerAppName, jobIdentifier, jobName, map)<br><br>  val jobConfigurationDetails: JobConfigurationDetails<br><br>  def consumeDataFromEEH(inputType: InputType.Type)(implicit sparkSession: SparkSession): DataFrame = {<br><br>    val sourceTopic = KafkaProperties.sourceTopic<br>    val startingOffset: String = &quot;latest&quot;<br><br>    val input = inputType<br><br>    try {<br>      val readStreamData = new ConsumerHelperBuilder(sparkSession, sourceTopic, JobType.Streaming)<br>        .withStartingOffsets(startingOffset)<br>        .withInputType(input)<br>        .withDeserializer(Deserializer.deserializerWithUdf)<br>        .build()<br>      readStreamData.consume().select(&quot;value&quot;)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Exception while fetch data from kafka topic :: Exception:&quot; + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br><br>        sparkSession.emptyDataFrame<br>    }<br>  }<br><br>  def convertToEntity(inputStr: String)(implicit sparkSession: SparkSession): MergedPricePlan = {<br>    try {<br>      log.info(&quot;Converting to Entity : convertToEntity()&quot;, customMeta)<br>      MergedPricePlan(JsonUtil.fromJson[PricePlan](inputStr), inputStr)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(&quot;Input event parsing error :: Exception :&quot; + exception.printStackTrace(), customMeta)<br>        new EventDeserException(&quot;400&quot;, exception).handle()<br>        MergedPricePlan(PricePlan(null, null, null), inputStr)<br>    }<br>  }<br><br>  def filterPropertyCode(inputDataset: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    log.info(&quot;Filtering property codes : filterPropertyCode()&quot;, customMeta)<br><br>    val column: Column = col(PricePlanPropertyCodeCode).alias(&quot;property-code&quot;)<br>    val (mgpPropertyDf, nonMgpPropertyDf) = FilterPropertyCodeUtil.filterPropertyCode(inputDataset.toDF(), column)<br><br>    // Directly convert DataFrame to Dataset without JSON conversion<br>    val mgpDS = mgpPropertyDf.as[MergedPricePlan]<br>    mgpDS<br>  }<br><br>  def mandatoryFieldValidation(inputDataset: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): (Dataset[MergedPricePlan], Dataset[MergedPricePlan]) = {<br>    log.info(&quot;Mandatory field validations : mandatoryFieldValidation()&quot;, customMeta)<br>    ///record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code<br>    val cleanDS = inputDataset.filter { rec =&gt;<br>      !(rec.pricePlan.propertyCode == null || rec.pricePlan.pricePlanDetails == null || rec.pricePlan.generationTime == null || rec.pricePlan.generationTime.isEmpty ||<br>        rec.pricePlan.propertyCode.code == null || rec.pricePlan.propertyCode.code.isEmpty || rec.pricePlan.propertyCode.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.code == null || rec.pricePlan.pricePlanDetails.code.isEmpty || rec.pricePlan.pricePlanDetails.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.isEmpty || rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.isEmpty) ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.exists(x =&gt; x.code.isEmpty || x.code == null || x.code == &quot;&quot;  || x.guestsByType.isEmpty))<br>        )<br>    }<br>    val errorDS = inputDataset.filter { rec =&gt;<br>      (rec.pricePlan.propertyCode == null || rec.pricePlan.pricePlanDetails == null || rec.pricePlan.generationTime == null || rec.pricePlan.generationTime.isEmpty ||<br>        rec.pricePlan.propertyCode.code == null || rec.pricePlan.propertyCode.code.isEmpty || rec.pricePlan.propertyCode.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.code == null || rec.pricePlan.pricePlanDetails.code.isEmpty || rec.pricePlan.pricePlanDetails.code == &quot;&quot; ||<br>        rec.pricePlan.pricePlanDetails.ratePeriods.exists(record =&gt; record.inventoryTypes.exists(x =&gt; x.code.isEmpty || x.code == null || x.code == &quot;&quot; || x.guestsByType.isEmpty)))<br>    }<br><br>    (cleanDS, errorDS)<br>  }<br><br>  def processPricePlanRecords(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    log.info(&quot;Started Processing the Records | processPricePlanRecords()&quot;, customMeta)<br>    val latestPricePlanDs = priceData<br>      .groupByKey(record =&gt; (record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>      .mapGroups { case (_, iterator) =&gt;<br>        val buffer = iterator.toBuffer<br>        val latestRecord = buffer.maxBy(_.pricePlan.generationTime)<br>        latestRecord<br>      }<br>    latestPricePlanDs<br>  }<br><br>  def chunkByInventory(pricePlanRecs: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    // Flatten the dataset<br>    try {<br>      val flattened = pricePlanRecs.flatMap { pricePlanElem =&gt;<br>        pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.flatMap { ratePeriod =&gt;<br>          ratePeriod.inventoryTypes.map { pricePoint =&gt;<br>            Flattened(pricePlanElem, ratePeriod, pricePoint)<br>          }<br>        }<br>      }<br><br>      // Group by inventoryTypeCode and reconstruct<br>      val grouped = flattened.groupBy(&quot;pricePlanMerged.pricePlan.propertyCode.code&quot;, &quot;pricePoint.code&quot;)<br>        .agg(<br>          collect_list(&quot;pricePlanMerged&quot;).as(&quot;pricePlansMerged&quot;),<br>          collect_list(struct(&quot;ratePeriod&quot;, &quot;pricePoint&quot;)).as(&quot;ratePeriodsAndPoints&quot;)<br>        ).as[(String, String, Seq[MergedPricePlan], Seq[(RatePeriod, PricePoint)])]<br><br>      // Reconstruct the PricePlan dataset<br>      val transformed = grouped.flatMap { case (propertyCode, inventoryTypeCode, pricePlansMerged, ratePeriodsAndPoints) =&gt;<br>        pricePlansMerged.distinct.map { pricePlanMerged =&gt;<br>          val updatedRatePeriods = ratePeriodsAndPoints.map { case (ratePeriod, pricePoint) =&gt;<br>            ratePeriod.copy(inventoryTypes = Seq(pricePoint.copy(code = inventoryTypeCode)))<br>          }.sortBy(_.timespan.get.start)<br>          //added sort by<br>          pricePlanMerged.copy(pricePlan = pricePlanMerged.pricePlan.copy(pricePlanDetails = pricePlanMerged.pricePlan.pricePlanDetails.copy(ratePeriods = updatedRatePeriods)))<br>        }<br>      }<br>      transformed}<br>      catch<br>      {<br>        case e: Exception =&gt;<br>          log.error(s&quot;An error occurred in chunkByInventory: ${e.getMessage}&quot;, customMeta)<br>          sparkSession.emptyDataset[MergedPricePlan]<br>      }<br><br>  }<br><br>  def generateKeyA(propertycode: String, pricePlanCode: String, inventoryTypeCode: String): String = {<br>    s&quot;$PriceDocPrefix$UnderScore$propertycode$UnderScore$pricePlanCode$UnderScore$inventoryTypeCode&quot;<br>  }<br><br>  def getNonDelDocDB(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): Dataset[PricePlanDocumentDBRecord] = {<br>    import sparkSession.implicits._<br>    try{<br>    val collectionName = jobConfigDetails.doucmentdbmap.get.get(PricePlanCollection).get<br><br>    def generateKeyB(propertycode: String, pricePlanCode: String): String = {<br>      s&quot;$PriceDocPrefix$UnderScore$propertycode$UnderScore$pricePlanCode&quot;<br>    }<br><br>    val keys = priceData.map(record =&gt; generateKeyB(record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code)).dropDuplicates()<br>      .collect()<br>    log.info(&quot;fetching ALL records based on documentDB based on propertyCode PricePlanCode| getNonDelDocDB()&quot;, customMeta)<br><br><br>    val priceDS: Dataset[PricePlanDocumentDBRecord] = if (keys.isEmpty) {<br><br>      sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    } else {<br>      val readers =<br>        keys.map { key =&gt;<br><br>      val read=prepareDocumentDBReaderWithFilter(<br>        pricePlanReaderBuilder,<br>        collectionName,<br>        classOf[PricePlanDocumentDBRecord],<br>        Some(col(&quot;_id&quot;)<br>          .rlike(key)<br>        )<br>      )<br>          read<br>    }<br>      val dataFrames = readers.flatMap { reader =&gt;<br>        val df = reader.as[PricePlanDocumentDBRecord]<br>        if (!df.isEmpty) Some(df) else None<br>      }<br>      dataFrames.reduceOption(_ union _).getOrElse(sparkSession.emptyDataset[PricePlanDocumentDBRecord])<br>    }<br>    val todayDate = DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_2).format(LocalDateTime.now())<br><br>    // Collect all inventory codes from price1<br>    val priceInventoryCodes = priceData.flatMap(_.pricePlan.pricePlanDetails.ratePeriods.flatMap(_.inventoryTypes.map(_.code)))<br>      .collect()<br>      .toSet<br>    val expiresUpdated = priceDS.map { doc =&gt;<br>      val inventoryCode = doc.pricePlanDetails.ratePeriods.headOption.flatMap(_.inventoryTypes.headOption.map(_.code))<br><br>      if (inventoryCode.exists(code =&gt; !priceInventoryCodes.contains(code))) {<br>        doc.copy(expiresAt = todayDate)<br>      } else {<br>        doc<br>      }<br>    }<br>    expiresUpdated}<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in getNonDelDocDB: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    }<br>  }<br><br>  def processNonDeleteRecords(priceData: Dataset[MergedPricePlan], priceDS: Dataset[PricePlanDocumentDBRecord])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails)<br>  : DataFrame = {<br><br>    try {<br>      log.info(s&quot;Inside method processNonDeleteFlow for Create|Update flow&quot;, customMeta)<br>      val collectionName = jobConfigDetails.doucmentdbmap.get.get(PricePlanCollection).get<br><br><br>      val deletedInventoryRecords = priceDS.filter(doc =&gt; doc.expiresAt != null)<br><br><br>      val nonDeleteInventoryRecords = priceDS.filter(doc =&gt; doc.expiresAt == null)<br><br>      deletedInventoryRecords.count()<br>      nonDeleteInventoryRecords.count()<br><br>      if (nonDeleteInventoryRecords.isEmpty) {<br>        val chunkedData = splitDOW(priceData)<br>        val destinationDS = (entityForLatestRecord(chunkedData)).union(deletedInventoryRecords)<br><br>        log.info(&quot;PropertyCode &amp; pricePlanCode does not found ; insert new record in documentDB | processNonDeleteFlow()&quot;, customMeta)<br><br>        BuilderUtil.insertDocumentDBRecords(destinationDS.toDF(), collectionName)<br><br>      } else {<br><br>        val thirtyDayFilteredPrice = cleanupThirtyDayPeriodOne(priceData)<br>        if (!thirtyDayFilteredPrice.isEmpty) {<br>          val chunkedData = splitDOW(thirtyDayFilteredPrice)<br>          val updatedChunkData = entityForLatestRecord(chunkedData)<br>          val finalDestinationDS = updatedChunkData.union(deletedInventoryRecords)<br>          writeToS3(finalDestinationDS)<br>          BuilderUtil.insertDocumentDBRecords(finalDestinationDS.toDF(), collectionName)<br>          log.info(&quot; Insert new Instance of key | processNonDeleteFlow&quot;, customMeta)<br>        }<br>      }<br><br>      priceDS.toDF()<br>    }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in getPriceRecords: ${e.getMessage}&quot;, customMeta)<br>        singleToRecoverable(priceData)<br>        sparkSession.emptyDataFrame<br>    }<br><br>  }<br><br>  def processDeletePricePlanDBRecords(pricePlanDS: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord], jobConfigDetails: JobConfigurationDetails): DataFrame = {<br>    try {<br>      import sparkSession.implicits._<br>      log.info(&quot;Processing Delete actionEnum for PricePlanRecords from PricePlan Collection | processDeletePricePlanDBRecords()&quot;, customMeta)<br><br>      val collectionName = jobConfigDetails.doucmentdbmap.get(PricePlanCollection)<br><br>      def generateKeys(propertyCode: String, pricePlanCode: String, inventoryTypeCode: String): String = {<br>        s&quot;$PriceDocPrefix$UnderScore$propertyCode$UnderScore$pricePlanCode$UnderScore$inventoryTypeCode&quot;<br>      }<br><br>      val keys = pricePlanDS.map(record =&gt; generateKeys(record.pricePlan.propertyCode.code, record.pricePlan.pricePlanDetails.code, record.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code)).dropDuplicates().collect()<br><br><br>      val priceDS: Dataset[PricePlanDocumentDBRecord] = if (keys.isEmpty) {<br><br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>      } else {<br><br>        val reader = prepareDocumentDBReaderWithFilter(<br>          pricePlanReaderBuilder,<br>          collectionName,<br>          classOf[PricePlanDocumentDBRecord],<br>          Some(col(&quot;_id&quot;)<br>            .isin(keys: _*))<br>        )<br><br>        reader.as[PricePlanDocumentDBRecord]<br>      }<br>      log.info(&quot;fetching records based on documentDB based on propertyCode,PricePlanCode and Inv Type Code | processDeleteRatePlanDBRecords()&quot;, customMeta)<br><br>      log.info(&quot; Writing old record to S3 | processDeleteRatePlanDBRecords&quot;, customMeta)<br>      writeToS3(priceDS)<br><br>      val currentDate = DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_2).format(LocalDateTime.now())<br>      val resultDFWithExpire = priceDS.withColumn(expiresAt, lit(currentDate))<br>      BuilderUtil.insertDocumentDBRecords(resultDFWithExpire, collectionName)<br>      log.info(&quot;Inserted the updated records with expiresAt into PricePlan Collection | processDeleteRatePlanDBRecords()&quot;, customMeta)<br><br>      resultDFWithExpire<br>    }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in processDeleteRatePlanDBRecords: ${e.getMessage}&quot;, customMeta)<br><br>        singleToRecoverable(pricePlanDS)<br>        sparkSession.emptyDataFrame<br>    }<br>  }<br><br>  def splitDOW(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br>    try{<br>    def dowToIndices(dow: Seq[String]): Set[Int] = {<br>      val dowMap = Map(&quot;Monday&quot; -&gt; 1, &quot;Tuesday&quot; -&gt; 2, &quot;Wednesday&quot; -&gt; 3, &quot;Thursday&quot; -&gt; 4, &quot;Friday&quot; -&gt; 5, &quot;Saturday&quot; -&gt; 6, &quot;Sunday&quot; -&gt; 7)<br>      dow.map(d =&gt; dowMap(d)).toSet<br>    }<br><br>    // Function to expand date range based on DOW<br>    def expandDateRange(start: String, end: String, dow: Set[Int]): Seq[(String, String)] = {<br><br>      val startDate = java.time.LocalDate.parse(start)<br><br>      //var endDate = java.time.LocalDate.parse(end)<br>      /* val endDate = Option(end) match {<br>         case Some(&quot;&quot;) =&gt; startDate.plusDays(30)<br>         case Some(endValue) =&gt; LocalDate.parse(endValue)<br>         case None =&gt; startDate.plusDays(30)<br>       }*/<br>      val endDate = Option(end).filter(_.nonEmpty).map(LocalDate.parse).getOrElse(startDate.plusYears(4))<br>      var currentStart = startDate<br>      var currentDOW = currentStart.getDayOfWeek.getValue<br>      val expandedRanges = scala.collection.mutable.ListBuffer.empty[(String, String)]<br>      while (!currentStart.isAfter(endDate)) {<br>        if (dow.contains(currentDOW)) {<br>          var currentEnd = currentStart<br>          while (!currentEnd.isAfter(endDate) &amp;&amp; dow.contains(currentEnd.getDayOfWeek.getValue)) {<br>            currentEnd = currentEnd.plusDays(1)<br>          }<br>          expandedRanges += ((currentStart.toString, currentEnd.minusDays(1).toString))<br>          currentStart = currentEnd<br>        } else {<br>          currentStart = currentStart.plusDays(1)<br>        }<br>        currentDOW = currentStart.getDayOfWeek.getValue<br>      }<br>      expandedRanges<br>    }<br><br>    //UDF<br>    val udfReg = udf((start: String, end: String, dow: Seq[String]) =&gt;<br>      expandDateRange(start, end, dowToIndices(dow))<br>    )<br>    val priceDataWithExpandedDates = priceData.flatMap(row =&gt; {<br>      val ratePeriods = row.pricePlan.pricePlanDetails.ratePeriods<br>      ratePeriods.flatMap(ratePeriod =&gt; {<br>        val start = ratePeriod.timespan.flatMap(_.start).getOrElse(&quot;&quot;)<br>        val end = ratePeriod.timespan.flatMap(_.end).getOrElse(&quot;&quot;)<br>        val dow = ratePeriod.activeDowEnum<br>        val dowIndices = dowToIndices(dow)<br>        val expandedRanges = expandDateRange(start, end, dowIndices)<br>        expandedRanges.map { case (newStart, newEnd) =&gt;<br>          val updatedTimespan = Timespan(Some(newStart), Some(newEnd), ratePeriod.timespan.flatMap(_.duration))<br>          val updatedRatePeriod = ratePeriod.copy(timespan = Some(updatedTimespan))<br>          row.copy(pricePlan = row.pricePlan.copy(pricePlanDetails = row.pricePlan.pricePlanDetails.copy(ratePeriods = Seq(updatedRatePeriod))))<br>        }<br>      })<br>    })<br><br>    val explodedDF = priceDataWithExpandedDates<br>      .withColumn(ratePeriod, explode(col(&quot;pricePlan.pricePlanDetails.ratePeriods&quot;)))<br>      .withColumn(inventoryTypeSeq, explode(col(&quot;ratePeriod.inventoryTypes&quot;)))<br>      .withColumn(inventoryType, col(&quot;inventoryTypeSeq.code&quot;))<br><br>    val groupedDF = explodedDF<br>      .groupBy(<br>        col(pricePlan_propertyCode_code).as(propertyCode),<br>        col(pricePlan_pricePlanDetails_code).as(pricePlanCode),<br>        col(&quot;inventoryType&quot;)<br>      )<br>      .agg(<br>        collect_list(ratePeriod).as(ratePeriods),<br>        first(pricePlan).as(pricePlan),<br>        first(sourceString).as(sourceString)<br>      )<br>      .withColumn(pricePlan, struct(<br>        col(pricePlan_propertyCode).as(propertyCode),<br>        col(pricePlan_generationTime).as(generationTime),<br>        struct(<br>          col(pricePlan_pricePlanDetails_code).as(code),<br>          col(pricePlan_pricePlanDetails_actionEnum).as(actionEnum),<br>          col(pricePlan_pricePlanDetails_statusEnum).as(statusEnum),<br>          col(pricePlan_pricePlanDetails_priceGridDetails).as(priceGridDetails),<br>          col(pricePlan_pricePlanDetails_yieldManagement).as(yieldManagement),<br>          col(ratePeriods).as(ratePeriods),<br>          col(pricePlan_pricePlanDetails_priceGridStayRestriction).as(priceGridStayRestriction),<br>          col(pricePlan_pricePlanDetails_referencePriceGridCode).as(referencePriceGridCode),<br>          col(pricePlan_pricePlanDetails_pricing).as(pricing),<br>          col(pricePlan_pricePlanDetails_classificationsEnum).as(classificationsEnum),<br>          col(pricePlan_pricePlanDetails_currencyCode).as(currencyCode)<br>        ).as(pricePlanDetails)<br>      ))<br><br><br>    groupedDF.select(<br>      col(pricePlan),<br>      col(sourceString)<br>    ).as[MergedPricePlan]<br><br><br>  }<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in splitDOW: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[MergedPricePlan]<br>    }}<br><br>  def comparePriceDatasets(merged: Dataset[MergedPricePlan], documentDb: Dataset[PricePlanDocumentDBRecord])(implicit spark: SparkSession): Dataset[PriceComparisonResult] = {<br><br>    import spark.implicits._<br>    log<br>      .info(&quot;Comparing PricePlan dataset with documentDB PricePlan Records | comparePriceDatasets()&quot;,<br>        customMeta<br>      )<br>    val joined = merged.joinWith(documentDb, merged(&quot;pricePlan.propertyCode.code&quot;) === documentDb(&quot;pricePlanDetails.propertyCode&quot;), joinType = &quot;left&quot;)<br>    val comparisonResult = joined.map { case (price, pricePlanDocumentDBRecord) =&gt; {<br><br>      val guestsByTypeComparison = (price.pricePlan.pricePlanDetails.priceGridDetails.get.freeGuestsByType == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.freeGuestsByType)<br><br>      val ratePeriodsComparison = price.pricePlan.pricePlanDetails.ratePeriods.zip(pricePlanDocumentDBRecord.pricePlanDetails.ratePeriods).forall {<br>        case (ratePeriod, ratePeriodDB) =&gt;<br>          ratePeriod.timespan == ratePeriodDB.timespan &amp;&amp;<br>            ratePeriod.activeDowEnum == ratePeriodDB.activeDowEnum &amp;&amp;<br>            ratePeriod.rounding == ratePeriodDB.rounding &amp;&amp;<br>            ratePeriod.amountAdd.get.value / math.pow(10, ratePeriod.amountAdd.get.numberOfDecimals.get.toInt) == ratePeriodDB.amountAdd.get.value &amp;&amp;<br>            ratePeriod.amountOff.get.value / math.pow(10, ratePeriod.amountOff.get.numberOfDecimals.get.toInt) == ratePeriodDB.amountOff.get.value &amp;&amp;<br>            ratePeriod.percentAdd == ratePeriodDB.percentAdd &amp;&amp;<br>            ratePeriod.percentOff == ratePeriodDB.percentOff &amp;&amp;<br>            ratePeriod.localAmountAdd.get.value / math.pow(10, ratePeriod.localAmountAdd.get.numberOfDecimals.get.toInt) == ratePeriodDB.localAmountAdd.get.value &amp;&amp;<br>            ratePeriod.localAmountOff.get.value / math.pow(10, ratePeriod.localAmountOff.get.numberOfDecimals.get.toInt) == ratePeriodDB.localAmountOff.get.value &amp;&amp;<br>            ratePeriod.level == ratePeriodDB.level &amp;&amp;<br>            ratePeriod.occupancy == ratePeriodDB.occupancy &amp;&amp;<br>            ratePeriod.guestTypeEnum == ratePeriodDB.guestTypeEnum &amp;&amp;<br>            ratePeriod.inventoryTypes.zip(ratePeriodDB.inventoryTypes).forall {<br>              case (invType, invTypeDB) =&gt;<br>                invType.code == invTypeDB.code &amp;&amp;<br>                  invType.monetaryAmount.get.value / math.pow(10, invType.monetaryAmount.get.numberOfDecimals.get.toInt) == invTypeDB.monetaryAmount.get.value &amp;&amp;<br>                  invType.localMonetaryAmount.get.value / math.pow(10, invType.localMonetaryAmount.get.numberOfDecimals.get.toInt) == invTypeDB.localMonetaryAmount.get.value &amp;&amp;<br>                  invType.changedInd == invTypeDB.hasChanged &amp;&amp;<br>                  invType.extraPersonCharge.get.value / math.pow(10, invType.extraPersonCharge.get.numberOfDecimals.get.toInt) == invTypeDB.extraPersonCharge.get.value<br>            }<br><br><br>      }<br><br><br>      val equal = (price.pricePlan.pricePlanDetails.code == pricePlanDocumentDBRecord.pricePlanDetails.code &amp;&amp;<br>        price.pricePlan.propertyCode.code == pricePlanDocumentDBRecord.pricePlanDetails.propertyCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code == pricePlanDocumentDBRecord.pricePlanDetails.inventoryCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.actionEnum == pricePlanDocumentDBRecord.pricePlanDetails.actionEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.statusEnum == pricePlanDocumentDBRecord.pricePlanDetails.statusEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.name == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.name &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.typeEnum == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.typeEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.taxInclusionEnum == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.taxInclusionEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.priceEndsWith == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.priceEndsWith &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productCategoryCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productCategoryCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productClassCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productClassCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.productCode == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.productCode &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults == pricePlanDocumentDBRecord.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults &amp;&amp;<br>        guestsByTypeComparison &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.yieldChangedInd == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.yieldChangedInd &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.typeEnum == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.typeEnum &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.flatAmount.get.value &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.percent == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.percent &amp;&amp;<br>        price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.get.toInt) == pricePlanDocumentDBRecord.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value &amp;&amp;<br>        ratePeriodsComparison &amp;&amp;<br>        price.pricePlan.pricePlanDetails.pricing == pricePlanDocumentDBRecord.pricePlanDetails.pricing &amp;&amp;<br>        price.pricePlan.pricePlanDetails.classificationsEnum == pricePlanDocumentDBRecord.pricePlanDetails.classificationsEnum<br><br>        )<br>      PriceComparisonResult(price, equal)<br>    }<br>    }<br>    log<br>      .info(&quot;Comparing PricePlan dataset with documentDB PricePlan Records Completed | comparePriceDatasets()&quot;,<br>        customMeta<br>      )<br>    comparisonResult<br>  }<br><br>  def entityForLatestRecord(priceData: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails)<br>  : Dataset[PricePlanDocumentDBRecord] = {<br>    import sparkSession.implicits._<br>    try{<br>    val destinationDS: Dataset[PricePlanDocumentDBRecord] = priceData.map { pricePlanElem =&gt;<br>      createPriceDocument(pricePlanElem, generateKeyA(pricePlanElem.pricePlan.propertyCode.code, pricePlanElem.pricePlan.pricePlanDetails.code, pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>    }<br>    destinationDS}<br>    catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in splitDOW: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[PricePlanDocumentDBRecord]<br>    }<br><br>  }<br><br>  def createPriceDocument(price: MergedPricePlan, key: String): PricePlanDocumentDBRecord = {<br><br><br>    val ratePeriodElements = scala.collection.mutable.ListBuffer[RatePeriodDB]()<br><br>    price.pricePlan.pricePlanDetails.ratePeriods.foreach(rate =&gt; {<br>      val counts = rate.inventoryTypes.flatMap(_.guestsByType.flatMap(_.count))<br>      val minDoubleRateCount = counts.filter(_ &gt;= 2)<br>      val singleOccupancyRate = if (counts.nonEmpty) Some(counts.min) else None<br>      val doubleOccupancyRate = if (minDoubleRateCount.nonEmpty) Some(minDoubleRateCount.min) else if (counts.contains(1)) Some(1) else None<br>      val currencyCdPricePlanDetails = price.pricePlan.pricePlanDetails.currencyCode<br><br>      val singleOccInv = rate.inventoryTypes.filter(_.guestsByType.get.count == singleOccupancyRate).head<br>      val doubleOccInv = rate.inventoryTypes.filter(_.guestsByType.get.count == doubleOccupancyRate).head<br>      val currencyCode = if (singleOccInv.monetaryAmount.isDefined &amp;&amp; singleOccInv.monetaryAmount.get.currencyCode.isDefined) singleOccInv.monetaryAmount.get.currencyCode.get else currencyCdPricePlanDetails<br><br>      ratePeriodElements += RatePeriodDB(<br>        rate.activeDowEnum, rate.rounding,<br>        tieredRates = None,<br>        if (singleOccInv.monetaryAmount.isDefined) {<br>          if (singleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            singleOccInv.monetaryAmount.get.value / math.pow(10, singleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            singleOccInv.monetaryAmount.get.value<br><br>        } else null,<br><br>        if (singleOccInv.monetaryAmount.isDefined) {<br>          if (singleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            singleOccInv.monetaryAmount.get.value / math.pow(10, singleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            singleOccInv.monetaryAmount.get.value<br><br>        } else null,<br><br>        if (doubleOccInv.monetaryAmount.isDefined) {<br>          if (doubleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            doubleOccInv.monetaryAmount.get.value / math.pow(10, doubleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            doubleOccInv.monetaryAmount.get.value<br><br>        } else null<br>        ,<br>         <br>        if (doubleOccInv.monetaryAmount.isDefined) {<br>          if (doubleOccInv.monetaryAmount.get.numberOfDecimals.isDefined)<br>            doubleOccInv.monetaryAmount.get.value / math.pow(10, doubleOccInv.monetaryAmount.get.numberOfDecimals.get.toInt)<br>          else<br>            doubleOccInv.monetaryAmount.get.value<br><br>        } else null<br>        ,<br>        if (rate.amountAdd.isDefined) {<br>          if (rate.amountAdd.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.amountAdd.get.value / math.pow(10, rate.amountAdd.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.amountAdd.get.value))<br>        } else None,<br>        if (rate.amountOff.isDefined) {<br>          if (rate.amountOff.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.amountOff.get.value / math.pow(10, rate.amountOff.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.amountOff.get.value))<br>        } else None,<br>        rate.percentAdd,<br>        rate.percentOff,<br>        if (rate.localAmountAdd.isDefined) {<br>          if (rate.localAmountAdd.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.localAmountAdd.get.value / math.pow(10, rate.localAmountAdd.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.localAmountAdd.get.value))<br>        } else None,<br>        if (rate.localAmountOff.isDefined) {<br>          if (rate.localAmountOff.get.numberOfDecimals.isDefined) {<br>            Some(CurrValue(rate.localAmountOff.get.value / math.pow(10, rate.localAmountOff.get.numberOfDecimals.get.toInt)))<br>          } else Some(CurrValue(rate.localAmountOff.get.value))<br>        } else None,<br>        rate.level,<br>        rate.occupancy,<br>        rate.guestTypeEnum,<br>        rate.inventoryTypes.map(<br>          invElem =&gt;<br>            PricePointDB(<br>              invElem.code,<br>              if (invElem.monetaryAmount.isDefined) {<br>                if (invElem.monetaryAmount.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.monetaryAmount.get.value / math.pow(10, invElem.monetaryAmount.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.monetaryAmount.get.value))<br>              } else None,<br>              if (invElem.localMonetaryAmount.isDefined) {<br>                if (invElem.localMonetaryAmount.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.localMonetaryAmount.get.value / math.pow(10, invElem.localMonetaryAmount.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.localMonetaryAmount.get.value))<br>              } else None,<br>              if (invElem.extraPersonCharge.isDefined) {<br>                if (invElem.extraPersonCharge.get.numberOfDecimals.isDefined) {<br>                  Some(CurrValue(invElem.extraPersonCharge.get.value / math.pow(10, invElem.extraPersonCharge.get.numberOfDecimals.get.toInt)))<br>                } else Some(CurrValue(invElem.extraPersonCharge.get.value))<br>              } else None,<br>              invElem.changedInd, rate.occupancy, rate.guestTypeEnum, invElem.guestsByType<br>            )<br>        ),<br>        rate.timespan,<br>        Option(currencyCode)<br>      )<br>    }<br>    )<br><br>    val pricePlanRecords = PricePlanDetailsDB(price.pricePlan.pricePlanDetails.code,<br>      price.pricePlan.propertyCode.code, price.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code,<br>      price.pricePlan.pricePlanDetails.actionEnum, price.pricePlan.pricePlanDetails.statusEnum, Some(<br>        PriceGridDetailsDB(<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.name, price.pricePlan.pricePlanDetails.priceGridDetails.get.typeEnum,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.applicableToAllInventoryTypesInd,<br>          if (price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.taxInclusionEnum,<br>          //price.pricePlan.pricePlanDetails.priceGridDetails.get.extraPersonCharge.get.currencyCode,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.flatMap(_.extraPersonCharge).map(_.currencyCode).getOrElse(null),<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.priceEndsWith, price.pricePlan.pricePlanDetails.priceGridDetails.get.productCategoryCode,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.productClassCode, price.pricePlan.pricePlanDetails.priceGridDetails.get.productCode,<br>          if (price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value / math.pow(10, price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.priceGridDetails.get.localExtraPersonCharge.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.maxFreeNonAdults,<br>          price.pricePlan.pricePlanDetails.priceGridDetails.get.freeGuestsByType<br><br>        )), Some(<br>        YieldManagementDB(<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.yieldChangedInd,<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.typeEnum,<br>          if (price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.flatAmount.get.value))<br>          } else None,<br>          price.pricePlan.pricePlanDetails.yieldManagement.get.percent,<br>          if (price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.isDefined) {<br>            if (price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.isDefined) {<br>              Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value / math.pow(10, price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.numberOfDecimals.get.toInt)))<br>            } else Some(CurrValue(price.pricePlan.pricePlanDetails.yieldManagement.get.localFlatAmount.get.value))<br>          } else None<br>        )),<br>      ratePeriodElements, price.pricePlan.pricePlanDetails.priceGridStayRestriction,<br>      price.pricePlan.pricePlanDetails.referencePriceGridCode, price.pricePlan.pricePlanDetails.pricing,<br>      price.pricePlan.pricePlanDetails.classificationsEnum, price.pricePlan.pricePlanDetails.currencyCode<br>    )<br><br>    PricePlanDocumentDBRecord(key, pricePlanRecords, null)<br>  }<br><br>  val avroSchema = new StructType()<br>    .add(&quot;timeStamp&quot;, StringType, nullable = true)<br>    .add(&quot;metadata&quot;, MapType(StringType, StringType, valueContainsNull = true))<br>    .add(&quot;retryCounter&quot;, IntegerType, nullable = true)<br>    .add(&quot;payLoad&quot;, StringType, nullable = true)<br>    .add(&quot;eventVersion&quot;, StringType, nullable = true)<br>    .add(&quot;eventSource&quot;, StringType, nullable = true)<br>    .add(&quot;destinationTopic&quot;, StringType, nullable = true)<br>    .add(&quot;eventIdentifier&quot;, StringType, nullable = true)<br>    .add(&quot;payLoadType&quot;, StringType, nullable = true)<br>    .add(&quot;recordNum&quot;, StringType, nullable = true)<br>    .add(&quot;error&quot;, StringType, nullable = true)<br>    .add(&quot;errorType&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;message&quot;, ArrayType(StringType, containsNull = true))<br>    .add(&quot;warning&quot;, ArrayType(StringType, containsNull = true))<br><br><br>  def produceDataToEEH(data: DataFrame)(implicit checkpointPath: Option[String]): Unit = {<br><br>    val destinationTopic = KafkaProperties.destinationTopic<br>    val schemaPath = &quot;merge-schema.avsc&quot;<br><br>    try {<br><br>      val producerHelper = new ProducerHelperBuilder(data, destinationTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(schemaPath))<br>        .withCaseClass(Some(classOf[EEHData]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>      log.info(&quot;Pushed into kafka topic&quot;, customMeta)<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br><br>  def writeDsToKafka(genTime: DataFrame, delRecs: DataFrame, nonDelRecs: DataFrame, pricePlanRecs: DataFrame)(implicit sparkSession: SparkSession,<br>                                                                                                              jobConfigurationDetails: JobConfigurationDetails): Dataset[String] = {<br>    val destinationTopic = jobConfigurationDetails.kafkatopicmap.get(&quot;DestinationTopic&quot;)<br>    val combinedRecs = delRecs.union(nonDelRecs)<br>    log<br>      .info(&quot;Kafka Writing records | writeDsToKafka()&quot;,<br>        customMeta<br>      )<br><br>    val combinedWithGenTime = combinedRecs.join(<br>      genTime,<br>      combinedRecs(&quot;pricePlanDetails.propertyCode&quot;) === genTime(&quot;propertyCode&quot;) &amp;&amp;<br>        combinedRecs(&quot;pricePlanDetails.code&quot;) === genTime(&quot;pricePlanCode&quot;),<br>      &quot;left&quot;<br>    ).withColumn(&quot;generationTime&quot;, coalesce(genTime(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>    val initialPricePlanExploded = pricePlanRecs.withColumn(propertyCode, col(&quot;pricePlan.propertyCode.code&quot;))<br>      .withColumn(pricePlanId, col(&quot;pricePlan.pricePlanDetails.code&quot;))<br>      .withColumn(actionEnum, col(&quot;pricePlan.pricePlanDetails.actionEnum&quot;))<br>      .withColumn(modifiedTimestamp, col(&quot;pricePlan.generationTime&quot;))<br>      .withColumn(ratePeriod, explode(col(&quot;pricePlan.pricePlanDetails.ratePeriods&quot;)))<br>      .withColumn(inventoryTypeSeq, explode(col(&quot;ratePeriod.inventoryTypes&quot;)))<br>      .withColumn(inventoryType, col(&quot;inventoryTypeSeq.code&quot;))<br>      .filter(not(col(actionEnum) === NoChange))<br>      .withColumn(isActive, when(col(actionEnum) === Delete, &quot;false&quot;)<br>        .otherwise(when(col(actionEnum) === Remove, &quot;false&quot;)<br>          .otherwise(&quot;true&quot;)))<br>      .select(propertyCode, pricePlanId, actionEnum, modifiedTimestamp, inventoryType, isActive)<br><br>    val pricePlanProcessedExploded = combinedWithGenTime.withColumn(propertyCode, col(&quot;pricePlanDetails.propertyCode&quot;))<br>      .withColumn(pricePlanId, col(&quot;pricePlanDetails.code&quot;))<br>      .withColumn(actionEnum, col(&quot;pricePlanDetails.actionEnum&quot;))<br>      .withColumn(modifiedTimestamp, col(&quot;generationTime&quot;))<br>      .withColumn(inventoryType, col(&quot;pricePlanDetails.inventoryCode&quot;))<br>      .filter(not(col(actionEnum) === NoChange)).dropDuplicates(&quot;propertyCode&quot;, &quot;pricePlanId&quot;, &quot;inventoryType&quot;)<br>      .withColumn(isActive, when(col(&quot;expiresAt&quot;).isNotNull, &quot;false&quot;).otherwise(&quot;true&quot;))<br>      .select(propertyCode, pricePlanId, actionEnum, modifiedTimestamp, inventoryType, isActive)<br>      .union(initialPricePlanExploded).dropDuplicates()<br><br>    val pricePlanExploded = pricePlanProcessedExploded<br>      .withColumn(&quot;mergerData&quot;,<br>        to_json(struct(propertyCode, pricePlanId, inventoryType, isActive, modifiedTimestamp)))<br>      .withColumn(EventSource, lit(jobName))<br>      .withColumn(EventVersion, lit(EmptyStr))<br>      .withColumn(EventIdentifier, lit(jobIdentifier))<br>      .withColumn(RetryCounter, lit(0))<br>      .withColumn(MetaData, typedLit(Map[String, String]()))<br>      .withColumn(&quot;DestinationTopic&quot;, lit(destinationTopic))<br>      .withColumn(RecordNum, lit(0))<br>      .withColumn(TimeStamp, current_timestamp())<br>      .withColumn(priceplan.PricePlanIngestion.Error, lit(EmptyStr))<br>      .withColumn(ErrorType, EmptyArray())<br>      .withColumn(Message, EmptyArray())<br>      .withColumn(Warning, EmptyArray())<br>      .withColumn(PayLoad, col(&quot;mergerData&quot;))<br>      .withColumn(PayLoadType, lit(Json))<br>      .withColumn(PricePlanEventJson,<br>        to_json(struct(TimeStamp, MetaData, RetryCounter, PayLoad, EventVersion, EventSource, &quot;DestinationTopic&quot;, EventIdentifier, PayLoadType,<br>          RecordNum, PricePlanIngestion.Error, ErrorType, Message, Warning)))<br>    import sparkSession.implicits._<br>    pricePlanExploded.select(&quot;PricePlanEventJson&quot;).write.format(&quot;console&quot;)<br>    pricePlanExploded.select(&quot;PricePlanEventJson&quot;).as[String]<br><br>  }<br><br><br>  def writeToS3(PricePlan: Dataset[PricePlanDocumentDBRecord])(implicit sparkSession: SparkSession, connectionConfigDetails: ConnectionConfigDetails, jobConfigDetails: JobConfigurationDetails): Boolean = {<br>    try {<br><br>      if (PricePlan.isEmpty) {<br>        log.info(&quot;DataFrame is empty. Not writing to S3.&quot;)<br>        return false<br>      }<br><br>      val s3bucket = connectionConfigDetails.bucketconfig.get(Default)<br><br>      log.info(&quot; inside writetos3 &quot;, customMeta)<br><br>      val S3PricePlanPath = S3Util.getCheckPointDirectory(s3bucket, jobName,<br>        jobConfigDetails.checkpointdirectory.get(pricePlanAuditBucket) + &quot;/&quot; + jobConfigDetails.checkpointdirectory.get(PricePlanS3backup))<br><br>      val S3FilePrefix = PricePlan.select(col(&quot;_id&quot;)).first().getString(0)<br>      val currentTimestamp = LocalDateTime.now.format(DateTimeFormatter.ofPattern(&quot;dd-MM-yyyy_HH_mm_ss&quot;))<br>      val S3PricePlanFilename = s&quot;$S3FilePrefix$UnderScore$currentTimestamp&quot;<br>      val todayDate = LocalDateTime.now.format(DateTimeFormatter.ofPattern(&quot;yyyyMMdd&quot;))<br>      val S3DayFolder = s&quot;$todayDate&quot;<br>      val s3Path = s&quot;$S3PricePlanPath/$S3DayFolder/$S3PricePlanFilename&quot;<br>      println(&quot;s3 path :&quot;, s3Path)<br><br>      PricePlan.withColumn(&quot;date&quot;, lit(S3DayFolder)).withColumn(&quot;propertyCode&quot;, col(&quot;pricePlanDetails.propertyCode&quot;))<br>        .write<br>        .format(&quot;json&quot;)<br>        .mode(&quot;append&quot;)<br>        .partitionBy(&quot;propertyCode&quot;, &quot;date&quot;)<br>        .save(s3Path)<br><br>      log.info(&quot;Written To S3&quot;,<br>        customMeta)<br>      false<br>    }<br>    catch {<br>      case exception: Exception =&gt; log.error(&quot;Error while writing to S3 in pricePlan :: exception :&quot; + exception.printStackTrace(),<br>        customMeta)<br>        new S3BucketException(exception,<br>          customMeta).handle()<br>        true<br>    }<br>  }<br><br><br>  def cleanupThirtyDayPeriodOne(pricePlan: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession, jobConfigDetails: JobConfigurationDetails): Dataset[MergedPricePlan] = {<br>    import sparkSession.implicits._<br><br>    val currentDate = LocalDate.now()<br>    val thirtyDayPeriod = currentDate.minusDays(30)<br><br>    log.info(&quot;inside cleanupThirtyDayPeriod process&quot;, customMeta)<br><br>    try {<br>      val pricePlansWithCondition = pricePlan.map { plan =&gt;<br>        val hasValidEndDate = plan.pricePlan.pricePlanDetails.ratePeriods.exists { ratePeriod =&gt;<br>          val endDate = ratePeriod.timespan.get.end match {<br>            case Some(date) =&gt; LocalDate.parse(date)<br>            case None =&gt; LocalDate.parse(ratePeriod.timespan.get.start.get).plusYears(4)<br>          }<br>          endDate.isAfter(thirtyDayPeriod)<br>        }<br>        (plan, hasValidEndDate)<br>      }<br><br>      val excludedPlans = pricePlansWithCondition.filter(!_._2).collect()<br><br><br>      val filteredPricePlans = pricePlansWithCondition.filter(_._2).map(_._1)<br><br><br>      log.info(&quot;cleanupThirtyDayPeriod process completed &quot;, customMeta)<br>      filteredPricePlans<br><br>    } catch {<br>      case e: Exception =&gt;<br>        log.error(s&quot;An error occurred in cleanupThirtyDayPeriod: ${e.getMessage}&quot;, customMeta)<br>        sparkSession.emptyDataset[MergedPricePlan]<br>    }<br>  }<br><br>  def writeIrrErrorToKafka(errorDS: Dataset[MergedPricePlan], topic: String, checkpoint: String): Unit = {<br>    try {<br>      log.info(&quot;Writing irrecoverable error to kafka : writeErrorToKafka()&quot;, customMeta)<br>      val checkPointDirectory = Some(checkpoint)<br>      val irrecError = errorDS.withColumn(&quot;errorCode&quot;, lit(&quot;400&quot;))<br>        .withColumn(&quot;exception&quot;, lit(&quot;exception due to Irrecoverable error&quot;))<br>        .withColumn(&quot;errorMessage&quot;, lit(&quot;error due to incoming message&quot;))<br>        .withColumn(&quot;mainTopicMessage&quot;, col(&quot;sourceString&quot;))<br>        .withColumn(&quot;error&quot;,<br>          struct(&quot;errorCode&quot;, &quot;exception&quot;, &quot;errorMessage&quot;, &quot;mainTopicMessage&quot;))<br>        .select(&quot;error&quot;)<br><br>      val errorFinal = irrecError.select(struct(&quot;*&quot;).alias(&quot;value&quot;))<br>        .withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br><br>      val schemaPath = &quot;irrecschema.avsc&quot;<br>      val producerHelper = new ProducerHelperBuilder(errorFinal, topic, jobTypes = JobType.Streaming)<br>        .withFilePath(Some(schemaPath))<br>        .withCheckPointPath(checkPointDirectory)<br>        .withCaseClass(Some(classOf[DLTError]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>    } catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br><br>  def singleToRecoverable(inputDS: Dataset[MergedPricePlan])(implicit sparkSession: SparkSession): Unit = {<br>    try {<br>      val finalDS = inputDS<br><br>        .select(col(&quot;sourceString&quot;)).coalesce(5).select(from_json(col(&quot;sourceString&quot;), Encoders.product[com.marriott.mmf.PricePlan].schema).as(&quot;value&quot;))<br><br><br>      val finalDF = finalDS.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br><br><br>      val producerHelper = new ProducerHelperBuilder(finalDF, KafkaProperties.recoverableErrorTopic, jobTypes = JobType.Batch)<br>        .withFilePath(Some(&quot;recschema.avsc&quot;))<br>        .withCaseClass(Some(classOf[com.marriott.mmf.PricePlan]))<br>        .withInputType(InputType.Avro)<br>        .withSerializer(Serializer.serializerWithAvroCloudEvent)<br>        .build()<br><br>      producerHelper.produce()<br>    }<br>    catch {<br>      case exception: Exception =&gt;<br>        log.error(ExceptionWhilePushingDataToKafka + exception.printStackTrace(),<br>          customMeta)<br>        new KafkaException(exception, customMeta).handle()<br>    }<br>  }<br>}<br><br><br>jobs<br><br>main.scala<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.job<br><br>import com.marriott.elevate.spark.framework.jobs.MgpApp<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.loggerAppName<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.CustomloggerMetadata<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.SparkConf<br><br>object Main extends MgpApp {<br>  val log: CustomLogger = new CustomLogger(this.getClass.getName)<br>  val customMeta: CustomloggerMetadata = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;,loggerAppName, jobConfigDetails.jobidentifier, jobConfigDetails.jobname, map)<br>  private val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br><br>  val pricePlanIngestionJob = new PricePlanIngestionJob(jobConfigurationDetails = jobConfigDetails,<br>    connectionConfigDetails = connectionConfigDetails,<br>    jobName = jobConfigDetails.jobname,<br>    jobIdentifier = jobConfigDetails.jobidentifier,<br>    triggerInterval = jobConfigDetails.triggerinterval.get)<br>  override def getSparkSession: Option[SparkConf] = Some(new SparkConf().set(&quot;spark.sql.streaming.kafka.useDeprecatedOffsetFetching&quot;, &quot;false&quot;).setAppName(&quot;PricePlanIngestion&quot;).setMaster(&quot;yarn&quot;))<br>  pricePlanIngestionJob.start()<br><br>}<br><br><br>priceplaningestionjob:<br><br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.job<br><br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.defaultBucket<br>import com.marriott.elevate.spark.framework.traits.StreamingJob<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion._<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity._<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper.PricePlanIngestionHelper<br>import com.marriott.rm.platform.constants.KafkaProperties<br>import com.marriott.rm.platform.enums.{InputType, OperationMode}<br>import com.marriott.rm.platform.util.ReaderBuilder<br>import com.marriott.utility.CustomLogger<br>import org.apache.spark.sql.functions.{col, from_json, lit}<br>import org.apache.spark.sql.streaming.Trigger<br>import org.apache.spark.sql.{Dataset, SparkSession}<br><br><br>class PricePlanIngestionJob(val jobConfigurationDetails: JobConfigurationDetails,<br>                            val connectionConfigDetails: ConnectionConfigDetails,<br>                            override val jobName: String,<br>                            override val jobIdentifier: String,<br>                            override val triggerInterval: Int)(implicit sparkSession: SparkSession)<br>  extends PricePlanIngestionHelper<br>    with StreamingJob<br>    with Serializable {<br><br>  this: PricePlanIngestionHelper =&gt;<br>  override val customMeta = CustomloggerMetadata(&quot;traceId&quot;, &quot;spanId&quot;, loggerAppName, jobIdentifier, jobName, map)<br>  override val log = new CustomLogger(this.getClass.getName)<br>  val map: Map[String, String] = Map(&quot;&quot; -&gt; &quot;&quot;)<br><br><br>  val documentDbDetails = connectionConfigDetails.documentconfig.get<br><br>  implicit val jobConfig: JobConfigurationDetails = jobConfigurationDetails<br><br><br>  val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)<br>  implicit val checkpointDirMap: Map[String, String] = jobConfigurationDetails.checkpointdirectory.get<br>  implicit val checkpointPath: Option[String] = Option(S3Util.getCheckPointDirectory(s3BucketName, jobName, checkpointDirMap.get(PricePlanIngestionCheckPoint).get))<br><br><br>  override def process(args: Map[String, String]): Unit = {<br>    /**<br>     * Consuming EEH Kafka topic data<br>     * Using UDF method, File is not required<br>     * */<br>    val data = consumeDataFromEEH(InputType.Avro)<br>    import sparkSession.implicits._<br>    val PricePlanDs = data.map(x =&gt; convertToEntity(x.getString(0)))<br>    val filteredDs = filterPropertyCode(PricePlanDs)<br><br>    val mandatoryFieldCheck = mandatoryFieldValidation(filteredDs)<br><br>    log.info(&quot;Mandatory attribute check has completed&quot;, customMeta)<br><br>    mandatoryFieldCheck._1.writeStream.foreachBatch { (batchDs: Dataset[MergedPricePlan], batchId: Long) =&gt;<br>      implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br><br>      val pricePlanInitial = processPricePlanRecords(batchDs)<br>      val pricePlanRecs = chunkByInventory(pricePlanInitial)<br>      val splitDOWRecs = splitDOW(pricePlanRecs)<br>      val genTime = pricePlanRecs.select(<br>        $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>        $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>        $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;)<br>      ).dropDuplicates(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;).toDF()<br><br>      val deleteFlow = pricePlanRecs.filter(_.pricePlan.pricePlanDetails.actionEnum.get == Delete)<br>      val delRecs = processDeletePricePlanDBRecords(deleteFlow)<br><br>      val nonDeleteFlow = pricePlanRecs.filter(_.pricePlan.pricePlanDetails.actionEnum.get != Delete)<br>      val initialNonDelRecs = getNonDelDocDB(nonDeleteFlow).persist()<br>      initialNonDelRecs.foreach(_ =&gt; ())<br>      processNonDeleteRecords(nonDeleteFlow, initialNonDelRecs)<br><br>      //here logic change required as non del flow is also having del,so use flag<br>      val flattenedJsonRecordsDataset = writeDsToKafka(genTime, delRecs, initialNonDelRecs.toDF(), splitDOWRecs.toDF())<br>      val jsonDF = flattenedJsonRecordsDataset.select(from_json(col(&quot;PricePlanEventJson&quot;), avroSchema).alias(&quot;value&quot;))<br>      val dataDF = jsonDF.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>        .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>        .withColumn(&quot;timestampType&quot;, lit(0))<br>      produceDataToEEH(dataDF)<br>      initialNonDelRecs.unpersist()<br>      (): Unit<br>    }<br>      .option(&quot;checkpointLocation&quot;, checkpointPath.get)<br>      .trigger(Trigger.ProcessingTime(triggerInterval))<br>      .start()<br><br>    val errorDS = PricePlanDs.filter(x =&gt; x.pricePlan.pricePlanDetails == null)<br><br>    writeIrrErrorToKafka(errorDS.union(mandatoryFieldCheck._2), KafkaProperties.irrecoverableErrorTopic, S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(defaultBucket), jobName,<br>      jobConfigurationDetails.checkpointdirectory.get(IrrecoverableErrorCheckpointDir)))<br><br><br>    sparkSession.streams.awaitAnyTermination<br><br>  }<br><br><br>}<br><br>util<br><br>udfs:<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.udfs<br><br>import java.sql.Date<br>import java.text.SimpleDateFormat<br>import java.util.Calendar<br>import org.apache.spark.sql.functions.udf<br><br>object SparkUdfs {<br>  lazy val EmptyArray = udf(() =&gt; Array.empty[Int])<br><br>  }<br><br><br><br><br>builderUtil.scala<br><br>package com.marriott.mgpapp.spark.priceplan.ingestion.util<br><br><br>import com.marriott.rm.platform.enums.OperationMode<br>import com.marriott.rm.platform.util.{MongoClientBuilder, MongoClientConnection, Reader, ReaderBuilder, WriterBuilder}<br>import com.mongodb.client.MongoClient<br>import com.mongodb.spark.sql.connector.config.WriteConfig.OperationType<br>import org.apache.spark.sql.functions.col<br>import org.apache.spark.sql.{Column, SparkSession}<br><br>import scala.reflect.runtime.universe.TypeTag<br>import org.apache.spark.sql.{Column, DataFrame, SparkSession}<br><br>object BuilderUtil {<br><br>  private var mongoClient: MongoClient = null<br><br>  /** Prepare and Get the Mongo Client<br>   *<br>   * @return - MongoClient<br>   */<br>  def getMongoClient(connection :MongoClientConnection = new MongoClientBuilder().build()): MongoClient = {<br>    if (mongoClient == null)<br>      mongoClient = connection.getMongoClient<br>    mongoClient<br>  }<br><br>  /** Prepare Document DB Reader With Filter for batch DS<br>   *<br>   * @param collectionName  - collection name<br>   * @param filter          - filter<br>   * @param collectionClass - case class for mapping<br>   * @param sparkSession    - Spark Session<br>   * @tparam T - Type of the Case Class<br>   * @return<br>   */<br>  def prepareDocumentDBReaderWithFilter[T &lt;: Product : TypeTag](<br>                                                                 readerBuilder: ReaderBuilder[T],<br>                                                                 collectionName: String,<br>                                                                 collectionClass: Class[T],<br>                                                                 filter: Option[Column] = None<br>                                                               )(implicit<br>                                                                 sparkSession: SparkSession<br>                                                               ): DataFrame = {<br><br>    val reader = readerBuilder<br>      .withCollection(collectionName)<br>      .withClass(collectionClass)<br><br>    if (filter.isDefined)<br>      reader<br>        .withApplyFilterColumn(true)<br>        .withFilterColumn(filter.get<br>          &amp;&amp; col(&quot;expiresAt&quot;).isNull<br>        )<br>        .build().read()<br>    else reader.build().read()<br>  }<br><br>  def insertDocumentDBRecords[T &lt;: Product : TypeTag](dataFrame: DataFrame,<br>                                                      collectionName: String,<br>                                                      operationType: OperationType = OperationType.REPLACE,<br>                                                      mode: OperationMode.Type = OperationMode.Batch<br>                                                     ) = {<br>    val writerBuilder = new WriterBuilder(dataFrame, mode)<br>      .withCollection(collectionName)<br>      .withOperationType(operationType)<br>    writerBuilder.build().write()<br>  }<br><br>}<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>priceplan_test<br><br><br><br><br>package com.marriott.mgpapp.spark.priceplan.test.helper<br>import com.marriott.elevate.spark.framework.config.{ConnectionConfigDetails, JobConfigurationDetails}<br>import com.marriott.elevate.spark.framework.constants.FrameworkConstants.{IrrecoverableErrorTopic, defaultBucket}<br>import com.marriott.elevate.spark.framework.utils.S3Util<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.IrrecoverableErrorCheckpointDir<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.{CurrValue, GuestsByType, MergedPricePlan, PriceComparisonResult, PriceGridDetailsDB, PricePlan, PricePlanDetails, PricePlanDetailsDB, PricePlanDocumentDBRecord, PricePointDB, Pricing, Range, RatePeriodDB, StayRestriction, TieredRate, Timespan, YieldManagementDB}<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper.PricePlanIngestionHelper<br>import com.marriott.mgpapp.spark.priceplan.ingestion.util.BuilderUtil.prepareDocumentDBReaderWithFilter<br>import com.marriott.mmf.{AmountRule, ClassificationEnum, CurrencyAmount, DependencyAttribute, Description, DowEnum, LOSPriceGrid, LOSPricing, LOSPricingActionEnum, Locale, MarketType, PriceGridTypeEnum, PriceRange, PricingFrequencyEnum, PropertyChain, PropertyCode, ProtectionLock, RatePeriod, RatePlanActionEnum, RoomBucket, SourceCRSEnum, TaxDetails, TaxInclusionEnum, TaxItem, TaxOverrideScope, TaxPeriod, TaxScope, TaxValue}<br>import com.marriott.rm.platform.enums.{InputType, OperationMode}<br>import com.marriott.rm.platform.util.{Reader, ReaderBuilder}<br>import com.marriott.rm.util.FilterPropertyCodeUtil<br>import org.apache.hadoop.security.UserGroupInformation<br>import org.apache.spark.sql.execution.streaming.MemoryStream<br>import org.apache.spark.sql.functions.{coalesce, col, from_json, lit}<br>import org.apache.spark.sql.types.StringType<br>import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession}<br>import org.mockito.ArgumentMatchers<br>import org.mockito.ArgumentMatchers.anyString<br>import org.mockito.Mockito.lenient<br>import org.mockito.scalatest.MockitoSugar<br>import org.scalatest.BeforeAndAfterAll<br>import org.scalatest.funsuite.AnyFunSuite<br>import org.scalatest.matchers.should.Matchers<br><br>import scala.reflect.runtime.universe.TypeTag<br>import com.marriott.mgpapp.spark.priceplan.ingestion.helper<br>case class MyClass(testColumn: String)<br>class PricePlanIngestionHelperTest extends AnyFunSuite<br>  with PricePlanIngestionHelper<br>  with TestHelper<br>  with java.io.Serializable<br>  with BeforeAndAfterAll<br>  with Matchers<br>  with MockitoSugar<br>{<br>  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(&quot;hduser&quot;))<br>  var sparkSession1: SparkSession = _<br>  var loadedDataFrame: DataFrame = _<br>  var inputDF: DataFrame = _<br><br><br>  override protected def beforeAll() {<br><br>    sparkSession1 = SparkSession<br>            /*.builder()<br>      .master(&quot;local[*]&quot;)<br>      .appName(&quot;Tests&quot;)<br>      .getOrCreate()*/<br>      .builder()<br>      .appName(&quot;PricePlanTest&quot;)<br>      .master(&quot;local&quot;)<br>      .config(&quot;spark.io.compression.codec&quot;, &quot;snappy&quot;)<br>      .getOrCreate()<br><br>  }<br><br>  override protected def afterAll {<br>    sparkSession1.stop()<br>  }<br><br>  def setupMockReaderBuilder[T &lt;: Product : TypeTag](expectedData: Seq[T]): ReaderBuilder[T] = {<br>    val mockReaderBuilder = mock[ReaderBuilder[T]]<br>    val mockReader = mock[Reader[T]]<br>    val expectedDF = sparkSession1.createDataFrame(expectedData)<br><br>    lenient().when(mockReaderBuilder.withCollection(anyString())).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withClass(any)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withApplyFilterColumn(anyBoolean)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.withFilterColumn(any)).thenReturn(mockReaderBuilder)<br>    lenient().when(mockReaderBuilder.build()).thenReturn(mockReader)<br>    lenient().when(mockReader.read()).thenReturn(expectedDF)<br><br>    mockReaderBuilder<br>  }<br><br>  def samplePricePlanDocumentDBData(_id: String, code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10), Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Some(Seq(tieredRate)), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-08-10&quot;), None)), Some(&quot;USD&quot;))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetailsDB(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue))), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;))), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }<br> /* def samplePricePlanMergedData(_id: String,code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10),Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Seq(tieredRate), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-08-10&quot;), None)))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetails(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2)))) ), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue)) ), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;)) ), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;)) ), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }*/<br><br>  def newSamplePricePlanDocumentDBData(_id: String,code: String, propCode: String, invCode: String): PricePlanDocumentDBRecord = {<br>    val currValue = CurrValue(BigDecimal(10.0))<br>    val tieredRate = TieredRate(BigDecimal(0), BigDecimal(10), BigDecimal(1), BigDecimal(10),Option(BigDecimal(10)))<br>    val pricePoint = PricePointDB(&quot;123&quot;, Some(currValue), Some(currValue), Some(currValue), Some(true), Some(2), Some(&quot;ADULT&quot;), Some(GuestsByType(Some(&quot;ADULT&quot;), Some(2))))<br>    val ratePeriod = RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;, &quot;WED&quot;, &quot;THU&quot;, &quot;FRI&quot;, &quot;SAT&quot;, &quot;SUN&quot;), Some(&quot;UP&quot;), Some(Seq(tieredRate)), BigDecimal(10.0), BigDecimal(10.0), BigDecimal(20.0), BigDecimal(20.0), Some(currValue), Some(currValue), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(currValue), Some(currValue), Some(&quot;BASE&quot;), Some(2), Some(&quot;ADULT&quot;), Seq(pricePoint), Some(Timespan(Some(&quot;2022-01-01&quot;), Some(&quot;2024-05-30&quot;), None)), Some(&quot;USD&quot;))<br>    PricePlanDocumentDBRecord(_id, PricePlanDetailsDB(code, propCode, invCode, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;), Some(PriceGridDetailsDB(Some(&quot;Standard&quot;), Some(&quot;FIXED&quot;), Some(true), Some(currValue), Some(&quot;EXCLUSIVE&quot;), Some(&quot;USD&quot;), Some(&quot;DAY&quot;), Some(&quot;ROOM&quot;), Some(&quot;STANDARD&quot;), Some(&quot;123&quot;), Some(currValue), Some(2), Seq(GuestsByType(Some(&quot;ADULT&quot;), Some(2)))) ), Some(YieldManagementDB(Some(true), Some(&quot;PERCENT&quot;), Some(currValue), Some(&quot;5&quot;), Some(currValue)) ), Seq(ratePeriod), Some(StayRestriction(Seq(Range(Some(&quot;2&quot;), Some(&quot;5&quot;))), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;), Seq(&quot;MON&quot;, &quot;TUE&quot;)) ), Some(&quot;REF123&quot;), Some(Pricing(Some(&quot;string&quot;), Some(&quot;string&quot;), Some(&quot;string&quot;)) ), Seq(&quot;MON&quot;, &quot;TUE&quot;), &quot;USD&quot;), &quot;24-04-2024 01:02:33&quot;)<br>  }<br><br>  test(&quot;writeToS3 should write Dataset to S3&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    val pricePlanDS = pricePlanDetailsData.toDS()<br><br>    implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(jobname = &quot;PricePlanIngestion&quot;, jobidentifier = &quot;PricePlan&quot;, triggerinterval = Some(1000), kafkamaxrequestsize = Some(20971520), checkpointdirectory = Some(Map(&quot;pricePlanAuditBucket&quot; -&gt; &quot;PricePlanIngestionCheckpointDir&quot;,&quot;PricePlanS3backup&quot;-&gt;&quot;PricePlanS3backup&quot;)), kafkatopicmap = Some(Map.empty), schemamap = Some(Map.empty))<br><br>    val result = writeToS3(pricePlanDS)<br>    assert(result == false)<br>  }<br>4<br>  test(&quot;writeToS3 should not handle empty datset&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    var pricePlanDS = Seq.empty[PricePlanDocumentDBRecord].toDS()<br>    val result = writeToS3(pricePlanDS)<br>    assert(result == false)<br><br>  }<br><br>  test(&quot;writeToS3 should not handle exception&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    val pricePlanDS = pricePlanDetailsData.toDS()<br><br>    implicit val connectionConfigDetails: ConnectionConfigDetails = ConnectionConfigDetails(null)<br>    implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(jobname = &quot;PricePlanIngestion&quot;, jobidentifier = &quot;PricePlan&quot;, triggerinterval = Some(1000), kafkamaxrequestsize = Some(20971520), checkpointdirectory = Some(Map(&quot;pricePlanAuditBucket&quot; -&gt; &quot;PricePlanIngestionCheckpointDir&quot;,&quot;PricePlanS3backup&quot;-&gt;&quot;PricePlanS3backup&quot;)), kafkatopicmap = Some(Map.empty), schemamap = Some(Map.empty))<br><br>    val result = writeToS3(pricePlanDS)<br>    assert(result == true)<br><br>  }<br><br><br>  test(&quot;prepareDocumentDBReaderWithFilter should work correctly&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val expectedData = Seq(<br>      MyClass(&quot;value1&quot;),<br>      MyClass(&quot;value2&quot;),<br>      MyClass(&quot;value3&quot;)<br>    )<br><br>    // Call the function with the mock ReaderBuilder<br>    val actualDF = prepareDocumentDBReaderWithFilter(setupMockReaderBuilder(expectedData), &quot;testCollection&quot;, classOf[MyClass])<br><br>    // Check that the returned DataFrame is as expected<br>    actualDF.collect() shouldEqual expectedData.toDF().collect()<br>  }<br><br><br>  test(&quot;cleanupThirtyDayPeriodOne should exclude plans with end date not after 30 days&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br>//sc<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br>//    val testData = Seq(sampleMergedPricePlanMessage)<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>    assert(result.count() == 0)<br>  }<br><br>  test(&quot;cleanupThirtyDayPeriodOne should return type PricePlanDocumentDB&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      samplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    //val pricePlanDS = pricePlanDetailsData.toDS()<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>    assert(result.isInstanceOf[Dataset[PricePlanDocumentDBRecord]])<br>  }<br><br>  test(&quot;cleanupThirtyDayPeriodOne should include plans with end date after 30 days&quot;) {<br><br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    val pricePlanDetailsData = Seq(<br>      newSamplePricePlanDocumentDBData(&quot;011&quot;, &quot;P11&quot;, &quot;PR001&quot;, &quot;I11&quot;))<br><br>    //val pricePlanDS = pricePlanDetailsData.toDS()<br>    val pricePlanDS = Seq(sampleMergedPricePlanMessage).toDS()<br><br>    val result = cleanupThirtyDayPeriodOne(pricePlanDS)<br><br>     assert(result.count() == 0)<br>  }<br><br>  test(&quot;Testing functionality of convertToEntity&quot;) {<br><br>    val inputDataStr = scala.io.Source.fromFile(&quot;PricePlanIngestion/src/test/resources/input.json&quot;).mkString<br>    val ite: MergedPricePlan = convertToEntity(inputDataStr)(sparkSession1)<br>    assert(ite != null)<br>  }<br><br>  test(&quot;Testing error scenario of convertToEntity&quot;) {<br>    implicit val spark = sparkSession1<br>    val inputDataStr = scala.io.Source.fromFile(&quot;PricePlanIngestion/src/test/resources/inputWrong.json&quot;).mkString<br>    val ite: MergedPricePlan = convertToEntity(inputDataStr)<br>    assert(ite.pricePlan == PricePlan(null, null, null))<br>  }<br><br><br>  test(&quot;filterPropertyCode&quot;) {<br>    implicit val spark = sparkSession1<br>    val testData = Seq(sampleMergedPricePlanMessage)<br>    import spark.implicits._<br>    val testDS = testData.toDS()<br>    withObjectMocked[FilterPropertyCodeUtil.type]{<br>      when(<br>        FilterPropertyCodeUtil.filterPropertyCode(<br>          ArgumentMatchers.any(classOf[DataFrame]),<br>          ArgumentMatchers.any(classOf[Column])<br>        )<br>      ).thenReturn((testDS.toDF(), testDS.toDF()))<br>      val res = filterPropertyCode(testDS)<br>      assert(res.isInstanceOf[Dataset[MergedPricePlan]])<br>    }<br><br>  }<br><br>  test(&quot;mandatoryFieldValidation_should validate records&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br>    val sampleData = Seq(sampleMergedPricePlanMessage).toDS()<br>    val result = mandatoryFieldValidation(sampleData)<br>    result._1.head().pricePlan.propertyCode.code shouldEqual &quot;BOGAR&quot;<br>  }<br><br><br><br>  test(&quot;consumeDataFromEEH should call readStreamWithParams with correct parameters&quot;) {<br>    implicit val spark = sparkSession1<br>    val result = consumeDataFromEEH(InputType.Avro)<br>    assert(result != null)<br>  }<br><br><br>  test(&quot;consumeDataFromEEH should throw error and return empty dataframe&quot;) {<br>    implicit val spark = sparkSession1<br>    val result = consumeDataFromEEH(InputType.Json)<br>    assert(result === spark.emptyDataFrame)<br>  }<br><br>  test(&quot;generateKeyA should generate key correctly&quot;) {<br><br>    val propertycode = &quot;property1&quot;<br>    val pricePlanCode = &quot;pricePlan1&quot;<br>    val inventoryCode = &quot;code1&quot;<br><br>    val key = generateKeyA(propertycode, pricePlanCode, inventoryCode)<br><br>    assert(key == &quot;PRICEPLANDOC_property1_pricePlan1_code1&quot;)<br>  }<br><br><br><br>      test(&quot;write DS to Kafka functionality should frame message correctly&quot;) {<br>        implicit val spark = sparkSession1<br>        import spark.implicits._<br><br>        // Provide an instance of JobConfigurationDetails<br>        implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br>        val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br><br>        val genTimeDataset = Seq(<br>          (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>        ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>        val delRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>        val delRecDataframe = delRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br><br>        val initialNonDelRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>        // val delRecDataframe = delRecsDataset.withColumn()<br>        val NonDelRecsDataframe = initialNonDelRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br>          val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br>        // Flatten the combinedRecs DataFrame<br>        val flattenedCombinedRecs = inputDataset.select(<br>          $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>          $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>          $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;),<br>          $&quot;sourceString&quot;<br>        )<br><br>        // Join the flattened DataFrame with genTime<br>        val combinedWithGenTime = flattenedCombinedRecs.join(<br>          genTimeDataset,<br>          flattenedCombinedRecs(&quot;propertyCode&quot;) === genTimeDataset(&quot;propertyCode&quot;) &amp;&amp;<br>            flattenedCombinedRecs(&quot;pricePlanCode&quot;) === genTimeDataset(&quot;pricePlanCode&quot;),<br>          &quot;left&quot;<br>        ).withColumn(&quot;generationTime&quot;, coalesce(genTimeDataset(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>        // Call the function under test<br>        val result = writeDsToKafka(genTimeDataset, delRecDataframe, NonDelRecsDataframe, pricePlanRecsDataset)(spark, jobConfigDetails)<br><br>        // Assert that the result is a Dataset of String<br>        assert(result.isInstanceOf[Dataset[String]])<br><br><br>      }<br>    test(&quot;create docDB entity&quot;) {<br>      val outPutDB = createPriceDocument(sampleMergedPricePlanMessage, generateKeyA(sampleMergedPricePlanMessage.pricePlan.propertyCode.code, sampleMergedPricePlanMessage.pricePlan.pricePlanDetails.code, sampleMergedPricePlanMessage.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.head.code))<br>      assert(outPutDB._id == &quot;PRICEPLANDOC_BOGAR_P1_ABC&quot;)<br>    }<br><br>    test(&quot;entityForLatestRecord 2 should transform PricePlan into PricePlanDocumentDBRecord&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br><br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br><br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br><br>      val outputDataset: Dataset[PricePlanDocumentDBRecord] = entityForLatestRecord(inputDataset)(spark, jobConfigDetails)<br><br>      assert(outputDataset.isInstanceOf[Dataset[PricePlanDocumentDBRecord]])<br>      //assert(outputDataset.head()._id != null)<br><br>    }<br><br>    test(&quot;compareRatesDatasets should compare two datasets correctly&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br>      val pricePlan1 = sampleMergedPricePlanMessage<br>      val pricePlanDoc1 = sampleDocPricePlan<br><br>      val merged = Seq(pricePlan1).toDS()<br>      val documentDb = Seq(pricePlanDoc1).toDS()<br><br>      val comparisonResult = comparePriceDatasets(merged, documentDb)(spark)<br><br>      assert(comparisonResult.isInstanceOf[Dataset[PriceComparisonResult]])<br><br>      val outputArray = comparisonResult.collect()<br><br>      assert(outputArray.length == 1)<br>      assert(outputArray.head.equals == true)<br>    }<br>  test(&quot;produceDataToEEH should produce data and log info&quot;) {<br>    implicit val spark = sparkSession1<br>    import spark.implicits._<br><br>    // Provide an instance of JobConfigurationDetails<br>    implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br><br>    val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br><br>    val genTimeDataset = Seq(<br>      (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>    ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br><br>    val delRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>    val delRecDataframe = delRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br><br>    val initialNonDelRecsDataset = sparkSession1.emptyDataset[PricePlanDocumentDBRecord]<br>    // val delRecDataframe = delRecsDataset.withColumn()<br>    val NonDelRecsDataframe = initialNonDelRecsDataset.withColumn(&quot;expiresAt&quot;, lit(&quot;&quot;))<br><br>    val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage2).toDS().toDF()<br><br><br>    // Flatten the combinedRecs DataFrame<br>    val flattenedCombinedRecs = inputDataset.select(<br>      $&quot;pricePlan.propertyCode.code&quot;.as(&quot;propertyCode&quot;),<br>      $&quot;pricePlan.pricePlanDetails.code&quot;.as(&quot;pricePlanCode&quot;),<br>      $&quot;pricePlan.generationTime&quot;.as(&quot;generationTime&quot;),<br>      $&quot;sourceString&quot;<br>    )<br><br>    // Join the flattened DataFrame with genTime<br>    val combinedWithGenTime = flattenedCombinedRecs.join(<br>      genTimeDataset,<br>      flattenedCombinedRecs(&quot;propertyCode&quot;) === genTimeDataset(&quot;propertyCode&quot;) &amp;&amp;<br>        flattenedCombinedRecs(&quot;pricePlanCode&quot;) === genTimeDataset(&quot;pricePlanCode&quot;),<br>      &quot;left&quot;<br>    ).withColumn(&quot;generationTime&quot;, coalesce(genTimeDataset(&quot;generationTime&quot;), lit(null).cast(StringType)))<br><br>    // Call the function under test<br>    val result = writeDsToKafka(genTimeDataset, delRecDataframe, NonDelRecsDataframe, pricePlanRecsDataset)(spark, jobConfigDetails).select(from_json(col(&quot;PricePlanEventJson&quot;), avroSchema).alias(&quot;value&quot;))<br>        val res2 = result.withColumn(&quot;key&quot;, lit(&quot;&quot;.getBytes)).withColumn(&quot;topic&quot;, lit(&quot;&quot;))<br>          .withColumn(&quot;partition&quot;, lit(0)).withColumn(&quot;offset&quot;, lit(0L)).withColumn(&quot;timestamp&quot;, lit(0L))<br>          .withColumn(&quot;timestampType&quot;, lit(0))<br><br><br>        //when(producerHelper.produce()).thenReturn(Some(ProducerOutput(Some(true), None)))<br><br>        //val producer = new Producer(dataFrame, producerHelper, logger, exceptionHandler)<br>        val a = produceDataToEEH(res2)<br><br>        assert(a != null)<br>        //  verify(logger).info(any[String], any[CustomMeta])<br>  }<br><br><br>    test(&quot;process priceplan batch for latest rec&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage, sampleMergedPricePlanMessage2).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processPricePlanRecords(inputDataset)<br><br>      assert(res != null)<br><br>    }<br><br><br>    test(&quot;process non del flow for new rec&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with same data&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with new data older than 30 days&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleNewDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process non del flow for updating rec with new data less than 30 days&quot;) {<br>      implicit val spark = SparkSession<br>        .builder()<br>        .master(&quot;local[*]&quot;)<br>        .appName(&quot;Test&quot;)<br>        .getOrCreate()<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq(sampleNewDocPricePlan)<br>      val inputDataset = Seq(sampleMergedPricePlanMessage2).toDS()<br>      val inputDataset2 = Seq(sampleNewDocPricePlan).toDS()<br><br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br><br>      val res = processNonDeleteRecords(inputDataset, inputDataset2)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;process del flow for new rec&quot;) {<br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br>      val expectedPricePlanData = Seq.empty[PricePlanDocumentDBRecord]<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      implicit val pricePlanReaderBuilder = setupMockReaderBuilder(expectedPricePlanData)<br><br>      val res = processDeletePricePlanDBRecords(inputDataset)<br><br>      assert(res != null)<br><br>    }<br><br>    test(&quot;Irrcoverable data should produce data and log info&quot;) {<br><br>      //val producerHelper = mock[ProducerHelper[EEHData]]<br><br><br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br><br><br>      // Create a MemoryStream of strings<br>      val stream = new MemoryStream[MergedPricePlan](1, spark.sqlContext)<br><br><br>      stream.addData(Seq(MergedPricePlan(PricePlan(null, null, null), &quot;abc&quot;)))<br><br>      // Create a DataFrame from the MemoryStream<br>      val invalidDF = stream.toDS()<br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>      //val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br><br>      val res = writeIrrErrorToKafka(invalidDF, jobConfigurationDetails.kafkatopicmap.get(IrrecoverableErrorTopic), S3Util.getCheckPointDirectory(connectionConfigDetails.bucketconfig.get(defaultBucket), jobName,<br>        jobConfigurationDetails.checkpointdirectory.get(IrrecoverableErrorCheckpointDir)))<br>      //when(producerHelper.produce()).thenReturn(Some(ProducerOutput(Some(true), None)))<br><br>      //val producer = new Producer(dataFrame, producerHelper, logger, exceptionHandler)<br>      assert(res != null)<br><br>      //  verify(logger).info(any[String], any[CustomMeta])<br>    }<br><br>    test(&quot;recoverable data should produce data and log info&quot;) {<br><br>      implicit val spark = sparkSession1<br>      import spark.implicits._<br>      val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>      // Provide an instance of JobConfigurationDetails<br>      implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>      val res = singleToRecoverable(inputDataset)(spark)<br>      assert(res != null)<br>    }<br><br>    test(&quot;object of source mmf topic&quot;) {<br>      implicit val spark = sparkSession1<br>      val res = com.marriott.mmf.PricePlan(<br>        PropertyChain(Some(&quot;chain&quot;), Some(&quot;MAR&quot;)), PropertyCode(Some(&quot;property&quot;), &quot;MCICO&quot;), Some(&quot;2024-05-03T07:57:15.627Z&quot;),<br>        SourceCRSEnum.ACRS,<br>        Some(LOSPricing(<br>          Some(LOSPricingActionEnum.Create), Some(&quot;basepricegrid&quot;), null, Some(Seq(<br>            LOSPriceGrid(null, Some(com.marriott.mmf.Range(null, null)))<br>          ))<br>        )),<br>        com.marriott.mmf.PricePlanDetails(<br>          Some(&quot;CAB1&quot;), Some(RatePlanActionEnum.Delete), null,<br>          Some(com.marriott.mmf.Promotion(<br>            null, null, null, Some(&quot;amountOff&quot;), null, null<br>          )<br>          ),<br>          Some(com.marriott.mmf.PriceGridDetails(<br>            null, Some(PriceGridTypeEnum.FlatRate), null,<br>            Some(TaxInclusionEnum.House), null,<br>            Some(CurrencyAmount(<br>              null, 12345, null, null<br>            )),<br>            Some(CurrencyAmount(<br>              null, 12345, null, null<br>            )),<br>            null, null, null, null,<br>            Some(Seq(<br>              AmountRule(null, null,<br>                Some(com.marriott.mmf.Timespan(Some(&quot;start&quot;), null, null)),<br>                Some(Seq(DowEnum.Friday)),<br>                Some(Seq(com.marriott.mmf.PricePoint(null, null, null, null, null,<br>                  Some(MarketType(null, Some(Description(<br>                    Some(Locale(<br>                      null, Some(&quot;USA&quot;)<br>                    )), Seq(&quot;abc&quot;), null<br>                  )), null)), null, null, null, null, null, null, null, Some(com.marriott.mmf.GuestsByType(Some(&quot;str&quot;), null))<br>                ))), null, null, null<br>              )<br>            )), null, null, null, null, null<br><br>          )<br>          ),<br>          Some(com.marriott.mmf.ProtectionLevel(<br>            Some(&quot;protectPricing&quot;), null<br>          )),<br>          Some(com.marriott.mmf.YieldManagement(<br>            null, null, null, null, Some(&quot;percent&quot;)<br>          )),<br>          Some(Seq(<br>            RatePeriod(<br>              null, null, Some(&quot;rounding&quot;), null, null, null, null, null, null, null, null, null, null, null<br>            )<br>          )),<br>          Some(com.marriott.mmf.StayRestriction(<br>            null, null, null, Some(&quot;invTypeCode&quot;), null, null, null, null, null, null, null, null, null, null, null, null,<br>            Some(RoomBucket(<br>              Some(&quot;roomBuck&quot;), null<br>            )), null, Some(Seq(<br>              ProtectionLock(<br>                Some(&quot;string&quot;), null<br>              )<br>            ))<br>          )),<br>          Some(&quot;refPriceGridCode&quot;),<br>          Some(com.marriott.mmf.TriggeringAction(<br>            null, Some(&quot;ratePlan&quot;), null, null, null<br>          )),<br>          Some(com.marriott.mmf.Pricing(<br>            Some(PricingFrequencyEnum.PerNight), null, null<br>          )),<br>          Some(Seq(<br>            TaxDetails(<br>              null, null, null, null, null, false,<br>              Some(DependencyAttribute(<br>                Some(&quot;attributeCode&quot;), null<br>              )), null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,<br>              null, null, null, null, null, null, Some(TaxScope(<br>                Some(PropertyChain(<br>                  Some(&quot;chain&quot;), null<br>                )), null<br>              )), Some(Seq(TaxItem(<br>                null, Some(&quot;CodeType&quot;), null<br>              ))), Some(Seq(TaxPeriod(<br>                null, null, Some(false), Some(Seq(TaxValue(<br>                  null, Some(&quot;percentVal&quot;), null, null, null, Some(Seq(PriceRange(<br>                    null, Some(CurrencyAmount(<br>                      null, 12345, null, null<br>                    ))<br>                  ))), null, null<br>                )))<br>              ))), null, Some(Seq(TaxOverrideScope(<br>                Some(&quot;code&quot;), null<br>              ))), null, null, null<br>            )<br>          )),<br>          Some(Seq(ClassificationEnum.Addon)),<br>          Some(&quot;currCode&quot;)<br>        )<br>      )<br>      assert(res.isInstanceOf[com.marriott.mmf.PricePlan])<br>    }<br>    test(&quot;amount rule = none &quot;) {<br>      implicit val spark = sparkSession1<br>      val res = com.marriott.mmf.PricePlan(chainCode = PropertyChain(), propertyCode = PropertyCode(code = &quot;code&quot;), pricePlanDetails = com.marriott.mmf.PricePlanDetails(), sourceCRSEnum = SourceCRSEnum.ACRS)<br>      val res1 = AmountRule()<br>      val res2 = CurrencyAmount(value = 12345)<br>      val res3 = DependencyAttribute()<br>      val res4 = Description(text = Seq(&quot;abc&quot;))<br>      val res5 = com.marriott.mmf.GuestsByType()<br>      val res6 = com.marriott.mmf.Locale()<br>      val res7 = LOSPriceGrid()<br>      val res8 = LOSPricing()<br>      val res9 = MarketType()<br>      val res10 = com.marriott.mmf.PriceGridDetails()<br>      val res11 = com.marriott.mmf.PricePoint()<br>      val res12 = com.marriott.mmf.PriceRange()<br>      val res13 = com.marriott.mmf.Pricing()<br>      val res14 = com.marriott.mmf.Promotion()<br>      val res15 = com.marriott.mmf.ProtectionLevel()<br>      val res16 = com.marriott.mmf.ProtectionLock()<br>      val res17 = com.marriott.mmf.Range()<br>      val res18 = com.marriott.mmf.RatePeriod()<br>      val res19 = com.marriott.mmf.RoomBucket()<br>      val res20 = com.marriott.mmf.StayRestriction()<br>      val res21 = com.marriott.mmf.TaxDetails(feeInd = true)<br>      val res22 = com.marriott.mmf.TaxItem()<br>      val res23 = com.marriott.mmf.TaxOverrideScope()<br>      val res24 = com.marriott.mmf.TaxPeriod()<br>      val res25 = com.marriott.mmf.TaxScope()<br>      val res26 = com.marriott.mmf.TaxValue()<br>      val res27 = com.marriott.mmf.Timespan()<br>      val res28 = com.marriott.mmf.TriggeringAction()<br>      val res29 = com.marriott.mmf.YieldManagement()<br>      assert(res.isInstanceOf[com.marriott.mmf.PricePlan])<br>      assert(res1.isInstanceOf[AmountRule])<br>      assert(res2.isInstanceOf[CurrencyAmount])<br>      assert(res3.isInstanceOf[DependencyAttribute])<br>      assert(res4.isInstanceOf[Description])<br>      assert(res5.isInstanceOf[com.marriott.mmf.GuestsByType])<br>      assert(res6.isInstanceOf[com.marriott.mmf.Locale])<br>      assert(res7.isInstanceOf[LOSPriceGrid])<br>      assert(res8.isInstanceOf[LOSPricing])<br>      assert(res9.isInstanceOf[MarketType])<br>      assert(res10.isInstanceOf[com.marriott.mmf.PriceGridDetails])<br>      assert(res11.isInstanceOf[com.marriott.mmf.PricePoint])<br>      assert(res12.isInstanceOf[com.marriott.mmf.PriceRange])<br>      assert(res13.isInstanceOf[com.marriott.mmf.Pricing])<br>      assert(res14.isInstanceOf[com.marriott.mmf.Promotion])<br>      assert(res15.isInstanceOf[com.marriott.mmf.ProtectionLevel])<br>      assert(res16.isInstanceOf[com.marriott.mmf.ProtectionLock])<br>      assert(res17.isInstanceOf[com.marriott.mmf.Range])<br>      assert(res18.isInstanceOf[com.marriott.mmf.RatePeriod])<br>      assert(res19.isInstanceOf[com.marriott.mmf.RoomBucket])<br>      assert(res20.isInstanceOf[com.marriott.mmf.StayRestriction])<br>      assert(res21.isInstanceOf[com.marriott.mmf.TaxDetails])<br>      assert(res22.isInstanceOf[com.marriott.mmf.TaxItem])<br>      assert(res23.isInstanceOf[com.marriott.mmf.TaxOverrideScope])<br>      assert(res24.isInstanceOf[com.marriott.mmf.TaxPeriod])<br>      assert(res25.isInstanceOf[com.marriott.mmf.TaxScope])<br>      assert(res26.isInstanceOf[com.marriott.mmf.TaxValue])<br>      assert(res27.isInstanceOf[com.marriott.mmf.Timespan])<br>      assert(res28.isInstanceOf[com.marriott.mmf.TriggeringAction])<br>      assert(res29.isInstanceOf[com.marriott.mmf.YieldManagement])<br><br>    }<br><br>  test(&quot;Split the data for Inventory Rate periods&quot;) {<br>    implicit val sparkSession = sparkSession1<br>    import sparkSession.implicits._<br>    implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br>    val sampleData = Seq(sampleMergedPricePlanChunkByInvMessage).toDS()<br>    val splitDOWRecs = splitDOW(sampleData)<br>    assert(!splitDOWRecs.filter(_.pricePlan.pricePlanDetails.ratePeriods.size &gt; 1).isEmpty)<br>  }<br><br>  test(&quot;Split the data for multiple Inventory &quot;) {<br>    implicit val sparkSession = sparkSession1<br>    import sparkSession.implicits._<br>    implicit val pricePlanReaderBuilder: ReaderBuilder[PricePlanDocumentDBRecord] = new ReaderBuilder[PricePlanDocumentDBRecord](sparkSession, OperationMode.Batch)<br>    val sampleData = Seq(updatedMergedPricePlanChunkByInvMessage).toDS()<br>    val splitByInv = chunkByInventory(sampleData)<br>    assert(!splitByInv.filter(_.pricePlan.pricePlanDetails.ratePeriods.head.inventoryTypes.size== 1).isEmpty)<br>  }<br><br>}<br>//  val genTime = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode1&quot;, &quot;pricePlanCode1&quot;, &quot;generationTime1&quot;)<br>//  )).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br>//<br>//  val delRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode1&quot;, &quot;pricePlanDetails1&quot;)<br>//  )).toDF(&quot;pricePlanDetails.propertyCode&quot;, &quot;pricePlanDetails.code&quot;)<br>//<br>//  val nonDelRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode2&quot;, &quot;pricePlanDetails2&quot;)<br>//  )).toDF(&quot;pricePlanDetails.propertyCode&quot;, &quot;pricePlanDetails.code&quot;)<br>//<br>//  val pricePlanRecs = spark.createDataFrame(Seq(<br>//    (&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;pricePlanDetails&quot;, &quot;generationTime&quot;)<br>//  )).toDF(&quot;pricePlan.propertyCode.code&quot;, &quot;pricePlan.pricePlanDetails.code&quot;, &quot;pricePlan.pricePlanDetails&quot;, &quot;pricePlan.generationTime&quot;)<br>//<br>//  implicit val jobConfigurationDetails: JobConfigurationDetails = mock[JobConfigurationDetails]<br>//  when(jobConfigurationDetails.kafkatopicmap).thenReturn(Some(Map(&quot;DestinationTopic&quot; -&gt; &quot;test-topic&quot;)))<br>//<br>//  //val kafkaProducer = new PricePlanIngestionHelper<br>//  val result: Dataset[String] = writeDsToKafka(genTime, delRecs, nonDelRecs, pricePlanRecs)<br>//<br>//  // Assertions<br>//  result.collect().foreach { jsonString =&gt;<br>//    jsonString should include(&quot;propertyCode&quot;)<br>//    jsonString should include(&quot;pricePlanId&quot;)<br>//    jsonString should include(&quot;isActive&quot;)<br>//  }}<br>//  test(&quot;write DS to kAfka functionality should frame message correctly&quot;) {<br>//    implicit val spark = sparkSession1<br>//    import spark.implicits._<br><br>// Provide an instance of JobConfigurationDetails<br>// implicit val jobConfigDetails: JobConfigurationDetails = jobConfigurationDetails<br>//    val inputDataset = Seq(sampleMergedPricePlanMessage).toDS()<br>//<br>//    // Dummy datasets for the other parameters<br>//    val genTimeDataset = Seq(<br>//      (&quot;BOGAR&quot;, &quot;P1&quot;, &quot;2021-01-01&quot;)<br>//    ).toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;generationTime&quot;)<br>//<br>//    val delRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().toDF()<br>//    val initialNonDelRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().toDF()<br>//    val pricePlanRecsDataset = Seq(sampleMergedPricePlanMessage).toDS().flatMap { pricePlanElem =&gt;<br>//      pricePlanElem.pricePlan.pricePlanDetails.ratePeriods.flatMap { ratePeriod =&gt;<br>//        ratePeriod.inventoryTypes.map { pricePoint =&gt;<br>//          (pricePlanElem.pricePlan.propertyCode.code,<br>//            pricePlanElem.pricePlan.pricePlanDetails.code,<br>//            ratePeriod,<br>//            pricePlanElem.pricePlan.generationTime,<br>//            pricePlanElem.sourceString)<br>//        }<br>//      }<br>//    }.toDF(&quot;propertyCode&quot;, &quot;pricePlanCode&quot;, &quot;ratePeriods&quot;, &quot;generationTime&quot;, &quot;sourceString&quot;)<br>//<br>//    // Call the function under test<br>//    val result = writeDsToKafka(genTimeDataset, delRecsDataset, initialNonDelRecsDataset, pricePlanRecsDataset)(spark, jobConfigDetails)<br>//<br>//    // Assert that the result is a Dataset of String<br>//    assert(result.isInstanceOf[Dataset[String]])<br><br><br><br><br><br><br><br><br><br><br><br><br>TestHelper:<br><br>package com.marriott.mgpapp.spark.priceplan.test.helper<br><br>import com.marriott.elevate.spark.framework.config._<br>import com.marriott.mgpapp.spark.priceplan.PricePlanIngestion.Default<br>import com.marriott.mgpapp.spark.priceplan.ingestion.entity.{CurrValue, CurrencyAmount, GuestsByType, MergedPricePlan, PriceGridDetails, PriceGridDetailsDB, PricePlan, PricePlanDetails, PricePlanDetailsDB, PricePlanDocumentDBRecord, PricePoint, PricePointDB, Pricing, PropertyCode, RatePeriod, RatePeriodDB, StayRestriction, TieredRate, Timespan, YieldManagement, YieldManagementDB}<br><br>import scala.collection.Seq<br>trait TestHelper {<br><br><br>  implicit val PricePlanIngestion = &quot;PricePlanIngestion&quot;<br>  val pricePlan = &quot;pricePlan&quot;<br>  implicit val SchemaName = &quot;SchemaName&quot;<br><br>  implicit val jobConfigurationDetails: JobConfigurationDetails = JobConfigurationDetails(<br>    jobname = PricePlanIngestion,<br>    jobidentifier = pricePlan,<br>    triggerinterval = Some(1000),<br>    kafkamaxrequestsize = Some(20971520),<br>    checkpointdirectory = Some(Map(&quot;ReservationFiltrationCheckpointDir&quot; -&gt; &quot;ReservationFiltrationCheckpointDir&quot;,<br>      &quot;ReservationS3backup&quot; -&gt; &quot;ReservationS3backup&quot;,<br>      &quot;IrrecoverableErrorCheckpointDir&quot; -&gt; &quot;IrrecoverableErrorCheckpointDir&quot;)),<br>    kafkatopicmap = Some(Map(&quot;DestinationTopic&quot; -&gt; &quot;crs.reservation.event&quot;,<br>      &quot;RecoverableErrorTopic&quot; -&gt; &quot;commerce.reservation.price-plan.sync.revenue-management.one-yield.recoverable-test.dlt&quot;,<br>      &quot;IrrecoverableErrorTopic&quot; -&gt; &quot;commerce.reservation.price-plan.sync.revenue-management.one-yield.irrecoverable-test.dlt&quot;)),<br>    schemamap = Some(Map(&quot;PricePlanBucket&quot; -&gt; &quot;RMS-Rates&quot;,&quot;PricePlanCollection&quot;-&gt;&quot;PricePlan&quot;)), // Empty map for now as no example provided in the configuration<br>    doucmentdbmap = Some(Map(&quot;documentcollectionname&quot; -&gt; &quot;PricePlan&quot;))<br>  )<br>  val jobName: String = &quot;PricePlanIngestion&quot;<br>  val jobIdentifier: String = &quot;PricePlanIngestion&quot;<br>  val triggerInterval: Int = 2000<br>  implicit val connectionConfigDetails: ConnectionConfigDetails =<br>    ConnectionConfigDetails(Some(DbDetails(<br>      jdbcurl = &quot;jdbc:mysql://localhost:3306/mydb&quot;,<br>      drivername = &quot;com.mysql.jdbc.Driver&quot;,<br>      username = &quot;my_username&quot;,<br>      password = &quot;my_password&quot;,<br>      poolsize = &quot;10&quot;,<br>      idletimeout = 10000, // Milliseconds<br>      jdbcurlro = &quot;jdbc:mysql://localhost:3306/mydb_read_only&quot;,<br>      drivernamero = &quot;com.mysql.jdbc.Driver&quot;,<br>      usernamero = &quot;ro_username&quot;,<br>      passwordro = &quot;ro_password&quot;,<br>      poolsizero = &quot;5&quot;,<br>      idletimeoutro = 1500, // Milliseconds<br>      jdbcurlwo = &quot;jdbc:mysql://localhost:3306/mydb_write_only&quot;,<br>      drivernamewo = &quot;com.mysql.jdbc.Driver&quot;,<br>      usernamewo = &quot;wo_username&quot;,<br>      passwordwo = &quot;wo_password&quot;,<br>      poolsizewo = &quot;8&quot;,<br>      idletimeoutwo = 12000L // Milliseconds<br>    )),<br>      Some(DocumentDbDetails(<br>        host = Seq(&quot;rms-oy-dev-documentdb.cluster-cbgemarietsq.us-east-1.docdb.amazonaws.com:27017&quot;).toList,<br>        username = &quot;user&quot;,<br>        password = &quot;password&quot;,<br>        db = &quot;RMS-RES-JOURNAL&quot;,<br>        collection = &quot;default&quot;,<br>        connectiontimeout = &quot;600000&quot;<br>      )),<br>      Some(MessageBrokerDetails(<br>        bootstrapserver = &quot;localhost:9092&quot;,<br>        sourcetopic = &quot;source_topic&quot;,<br>        destinationtopic = &quot;destination_topic&quot;,<br>        recoverableerrortopic = &quot;recoverable_error_topic&quot;,<br>        irrecoverableerrortopic = &quot;irrecoverable_error_topic&quot;,<br>        kafkakeyserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;,<br>        kafkavalueserializer = &quot;org.apache.kafka.common.serialization.StringSerializer&quot;,<br>        maxOffsetTriggerInterval = Some(1000), // Some specific value (milliseconds)<br>        kafkasecurityprotocol = None, // Using the default value (None)<br>        kafkassltruststorelocation = None, // Using the default value (None)<br>        kafkasslkeystorelocation = None, // Using the default value (None)<br>        kafkassltruststorepassword = None, // Using the default value (None)<br>        kafkasslkeystorepassword = None, // Using the default value (None)<br>        kafkaidentificationalgorithm = None, // Using the default value (None)<br>        kafkatruststoretype = None, // Using the default value (None)<br>        pollTimeoutMs = Some(5000) // Some specific value (milliseconds)<br>      )),<br>      Some(Map(&quot;default&quot; -&gt; &quot;Checkpoint\\Bucket&quot;)), Some(Map()), Some(Map()))<br><br>  val s3BucketName = connectionConfigDetails.bucketconfig.get(Default)<br>  implicit val checkpointDirMap: Map[String, String] = jobConfigurationDetails.checkpointdirectory.get<br>  implicit val checkpointPath: Option[String] = Some(&quot;Checkpoint\\Bucket&quot;)<br><br><br><br><br>  val sampleMergedPricePlanMessage2 = MergedPricePlan(PricePlan(<br>    propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>    generationTime = &quot;2021-01-02&quot;,<br>    PricePlanDetails(<br>      &quot;P1&quot;,<br>      Some(&quot;CREATE&quot;),<br>      Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetails(Some(&quot;Standard&quot;), Some(&quot;Regular&quot;), Some(true), Some(&quot;Inclusion 1&quot;), Some(&quot;Category 1&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;priceendswith&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(1), Seq(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagement(Some(false), Some(&quot;type&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;percentage&quot;))),<br>      Seq(RatePeriod(Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>        Some(Timespan(Some(&quot;start&quot;), Some(&quot;2024-05-30&quot;), Some(&quot;duration&quot;))),<br>        Some(&quot;down&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;10&quot;),<br>        Some(&quot;10&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;level&quot;),<br>        Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>        Some(1),<br>        Some(&quot;guestTypeEnum&quot;), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>      &quot;USD&quot;<br>    )<br>  ), &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;)<br><br><br>  val sampleDocPricePlan = PricePlanDocumentDBRecord(&quot;PRICEPLANNDOC_BOGAR_P1_ABC&quot;,<br>    PricePlanDetailsDB(&quot;P1&quot;,&quot;BOGAR&quot;,&quot;ABC&quot;,Some(&quot;CREATE&quot;),Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetailsDB(Some(&quot;Standard&quot;),Some(&quot;Regular&quot;),Some(true),Some(CurrValue(12.34)),Some(&quot;Inclusion 1&quot;),Some(&quot;usd&quot;),Some(&quot;priceendswith&quot;),Some(&quot;Category 1&quot;),Some(&quot;classCode&quot;),Some(&quot;pCode&quot;),Some(CurrValue(12.34)),Some(1),List(GuestsByType(Some(&quot;guest&quot;),Some(1)))) ),<br>      Some(YieldManagementDB(Some(false),Some(&quot;type&quot;),Some(CurrValue(12.34)),Some(&quot;percentage&quot;),Some(CurrValue(12.34))) ),<br>      Seq(RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;),Some(&quot;down&quot;),Some(Seq(TieredRate(null,null,null,null,null))),12.34,12.34,null,null,Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(&quot;10&quot;),Some(&quot;10&quot;),Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(&quot;level&quot;),Some(1),Some(&quot;guestTypeEnum&quot;),<br>        Seq(PricePointDB(&quot;ABC&quot;,Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(CurrValue(12.34)),Some(false),Some(1),Some(&quot;guestTypeEnum&quot;),Some(GuestsByType(Some(&quot;guest&quot;),Some(1))))),Some(Timespan(Some(&quot;start&quot;),Some(&quot;end&quot;),Some(&quot;duration&quot;))), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq( com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;),Some(&quot;max&quot;))),Seq(&quot;string&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;)) ),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;),Some(&quot;methods&quot;),Some(&quot;book&quot;)) ),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),&quot;USD&quot;),null)<br><br>  val sampleNewDocPricePlan = PricePlanDocumentDBRecord(&quot;PRICEPLANNDOC_BOGAR_P1_ABC&quot;,<br>    PricePlanDetailsDB(&quot;P1&quot;, &quot;BOGAR&quot;, &quot;ABC&quot;, Some(&quot;CREATE&quot;), Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetailsDB(Some(&quot;New&quot;), Some(&quot;Regular&quot;), Some(true), Some(CurrValue(12.34)), Some(&quot;Inclusion 1&quot;), Some(&quot;usd&quot;), Some(&quot;priceendswith&quot;), Some(&quot;Category 1&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(CurrValue(12.34)), Some(1), List(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagementDB(Some(false), Some(&quot;type&quot;), Some(CurrValue(12.34)), Some(&quot;percentage&quot;), Some(CurrValue(12.34)))),<br>      Seq(RatePeriodDB(Seq(&quot;MON&quot;, &quot;TUE&quot;), Some(&quot;down&quot;), Some(Seq(TieredRate(null, null, null, null,null))), 12.34, 12.34, null, null, Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(&quot;10&quot;), Some(&quot;10&quot;), Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(&quot;level&quot;), Some(1), Some(&quot;guestTypeEnum&quot;),<br>        Seq(PricePointDB(&quot;ABC&quot;, Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(CurrValue(12.34)), Some(false), Some(1), Some(&quot;guestTypeEnum&quot;), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))), Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;), &quot;USD&quot;), null)<br><br>  val sampleDelMessage = PricePlan(<br>    propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>    generationTime = &quot;2021-01-01&quot;,<br>    PricePlanDetails(<br>      &quot;P1&quot;,<br>      Some(&quot;Delete&quot;),<br>      Some(&quot;ACTIVE&quot;),<br>      Some(PriceGridDetails(Some(&quot;Standard&quot;), Some(&quot;Regular&quot;), Some(true), Some(&quot;Inclusion 1&quot;), Some(&quot;Category 1&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;priceendswith&quot;), Some(&quot;classCode&quot;), Some(&quot;pCode&quot;), Some(1), Seq(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>      Some(YieldManagement(Some(false), Some(&quot;type&quot;), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(&quot;percentage&quot;))),<br>      Seq(RatePeriod(Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>        Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))),<br>        Some(&quot;down&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;10&quot;),<br>        Some(&quot;10&quot;),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>        Some(&quot;level&quot;),<br>        Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>        Some(1),<br>        Some(&quot;guestTypeEnum&quot;), Some(&quot;USD&quot;))),<br>      Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>      Some(&quot;refPriceGridCode&quot;),<br>      Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>      Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>      &quot;USD&quot;<br>    )<br>  )<br><br><br><br><br><br><br><br>  val sampleMergedPricePlanMessage = MergedPricePlan(<br>     PricePlan(<br>      propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>      generationTime = &quot;2021-01-01&quot;,<br>      pricePlanDetails = PricePlanDetails(<br>        code = &quot;P1&quot;,<br>        actionEnum = Some(&quot;CREATE&quot;),<br>        statusEnum = Some(&quot;ACTIVE&quot;),<br>        priceGridDetails = Some(PriceGridDetails(<br>          name = Some(&quot;Standard&quot;),<br>          typeEnum = Some(&quot;Regular&quot;),<br>          applicableToAllInventoryTypesInd = Some(true),<br>          taxInclusionEnum = Some(&quot;Inclusion 1&quot;),<br>          productCategoryCode = Some(&quot;Category 1&quot;),<br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          priceEndsWith = Some(&quot;priceendswith&quot;),<br>          productClassCode = Some(&quot;classCode&quot;),<br>          productCode = Some(&quot;pCode&quot;),<br>          maxFreeNonAdults = Some(1),<br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br>        )),<br>         Some(YieldManagement(<br>          yieldChangedInd = Some(false),<br>          typeEnum = Some(&quot;type&quot;),<br>          flatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percent = Some(&quot;percentage&quot;)<br>        )),<br>        Seq(RatePeriod(<br>          activeDowEnum = Seq(&quot;MON&quot;, &quot;TUE&quot;),<br>          timespan = Some(Timespan(Some(&quot;start&quot;), Some(&quot;end&quot;), Some(&quot;duration&quot;))),<br>          rounding = Some(&quot;down&quot;),<br>          amountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          amountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percentAdd = Some(&quot;10&quot;),<br>          percentOff = Some(&quot;10&quot;),<br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localAmountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          level = Some(&quot;level&quot;),<br>          inventoryTypes = Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>          occupancy = Some(1),<br>          guestTypeEnum = Some(&quot;guestTypeEnum&quot;),<br>          Some(&quot;USD&quot;)<br>        )),<br>         Some(StayRestriction(Seq( com.marriott.mgpapp.spark.priceplan.<br>          ingestion.entity.Range(Some(&quot;min&quot;),Some(&quot;max&quot;))),Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;),Seq(&quot;String&quot;),Seq(&quot;String&quot;) )),<br>        referencePriceGridCode = Some(&quot;refPriceGridCode&quot;),<br>        pricing = Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>        classificationsEnum = Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>        currencyCode = &quot;USD&quot;<br>      )<br>    ),<br>    &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;<br>  )<br><br>  val sampleMergedPricePlanChunkByInvMessage = MergedPricePlan(<br>    PricePlan(<br>      propertyCode = PropertyCode(&quot;BOGAR&quot;),<br>      generationTime = &quot;2021-01-01&quot;,<br>      pricePlanDetails = PricePlanDetails(<br>        code = &quot;P1&quot;,<br>        actionEnum = Some(&quot;CREATE&quot;),<br>        statusEnum = Some(&quot;ACTIVE&quot;),<br>        priceGridDetails = Some(PriceGridDetails(<br>          name = Some(&quot;Standard&quot;),<br>          typeEnum = Some(&quot;Regular&quot;),<br>          applicableToAllInventoryTypesInd = Some(true),<br>          taxInclusionEnum = Some(&quot;Inclusion 1&quot;),<br>          productCategoryCode = Some(&quot;Category 1&quot;),<br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          priceEndsWith = Some(&quot;priceendswith&quot;),<br>          productClassCode = Some(&quot;classCode&quot;),<br>          productCode = Some(&quot;pCode&quot;),<br>          maxFreeNonAdults = Some(1),<br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br>        )),<br>        Some(YieldManagement(<br>          yieldChangedInd = Some(false),<br>          typeEnum = Some(&quot;type&quot;),<br>          flatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percent = Some(&quot;percentage&quot;)<br>        )),<br>        Seq(RatePeriod(<br>          activeDowEnum = Seq(&quot;Monday&quot;, &quot;Tuesday&quot;),<br>          timespan = Some(Timespan(Some(&quot;2025-07-20&quot;), Some(&quot;2025-08-10&quot;), Some(&quot;10&quot;))),<br>          rounding = Some(&quot;down&quot;),<br>          amountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          amountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          percentAdd = Some(&quot;10&quot;),<br>          percentOff = Some(&quot;10&quot;),<br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          localAmountOff = Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))),<br>          level = Some(&quot;level&quot;),<br>          inventoryTypes = Seq(PricePoint(&quot;ABC&quot;, Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(CurrencyAmount(Some(&quot;usd&quot;), 1234, Some(false), Some(2))), Some(false), Some(GuestsByType(Some(&quot;guest&quot;), Some(1))))),<br>          occupancy = Some(1),<br>          guestTypeEnum = Some(&quot;guestTypeEnum&quot;),<br>          Some(&quot;USD&quot;)<br>        )),<br>        Some(StayRestriction(Seq(com.marriott.mgpapp.spark.priceplan.<br>          ingestion.entity.Range(Some(&quot;min&quot;), Some(&quot;max&quot;))), Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;), Seq(&quot;String&quot;), Seq(&quot;String&quot;))),<br>        referencePriceGridCode = Some(&quot;refPriceGridCode&quot;),<br>        pricing = Some(Pricing(Some(&quot;freq&quot;), Some(&quot;methods&quot;), Some(&quot;book&quot;))),<br>        classificationsEnum = Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;),<br>        currencyCode = &quot;USD&quot;<br>      )<br>    ),<br>    &quot;{\&quot;chainCode\&quot;: {\&quot;chainName\&quot;: null, \&quot;code\&quot;: \&quot;ABC\&quot;}, \&quot;propertyCode\&quot;: {\&quot;hotelName\&quot;: null, \&quot;code\&quot;: \&quot;DPSEL\&quot;}, \&quot;generationTime\&quot;: \&quot;2024-04-30T13:10:43.080Z\&quot;, \&quot;sourceCRSEnum\&quot;: \&quot;ACRS\&quot;, \&quot;losPricing\&quot;: {\&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;basePriceGrid\&quot;: \&quot;PG001\&quot;, \&quot;enableInd\&quot;: true, \&quot;losPriceGrids\&quot;: [{\&quot;code\&quot;: \&quot;PG001\&quot;, \&quot;lengthOfStay\&quot;: {\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}}]}, \&quot;pricePlanDetails\&quot;: {\&quot;code\&quot;: \&quot;DPP2\&quot;, \&quot;actionEnum\&quot;: \&quot;Create\&quot;, \&quot;statusEnum\&quot;: \&quot;Active\&quot;, \&quot;bogo\&quot;: {\&quot;buyNumber\&quot;: 0, \&quot;getNumber\&quot;: 0, \&quot;maxNumber\&quot;: 0, \&quot;amountOff\&quot;: \&quot;10\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;priceGridCode\&quot;: null}, \&quot;priceGridDetails\&quot;: {\&quot;name\&quot;: \&quot;New price grid\&quot;, \&quot;typeEnum\&quot;: \&quot;FlatRate\&quot;, \&quot;applicableToAllInventoryTypesInd\&quot;: true, \&quot;taxInclusionEnum\&quot;: \&quot;IncludedInRate\&quot;, \&quot;productCategoryCode\&quot;: \&quot;string\&quot;, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localExtraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: \&quot;USD\&quot;, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonChargeMethodEnum\&quot;: \&quot;PerPricePlan\&quot;, \&quot;priceEndsWith\&quot;: \&quot;30\&quot;, \&quot;productClassCode\&quot;: \&quot;SPA\&quot;, \&quot;productCode\&quot;: \&quot;MASSAGE\&quot;, \&quot;amountRules\&quot;: null, \&quot;maxFreeNonAdults\&quot;: 0, \&quot;freeGuestsByType\&quot;: [{\&quot;guestType\&quot;: \&quot;Teen\&quot;, \&quot;count\&quot;: 2}], \&quot;rmsAllocationAllowedInd\&quot;: true, \&quot;nonRMSAllocationAllowedInd\&quot;: true, \&quot;deleteAtHotelAllowedInd\&quot;: true}, \&quot;protectionLevels\&quot;: {\&quot;protectPricing\&quot;: \&quot;basePrices\&quot;, \&quot;protectedValues\&quot;: [\&quot;allPrices\&quot;]}, \&quot;yieldManagement\&quot;: {\&quot;yieldChangedInd\&quot;: true, \&quot;typeEnum\&quot;: \&quot;AdjustmentValue\&quot;, \&quot;flatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localFlatAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percent\&quot;: \&quot;0.0\&quot;}, \&quot;ratePeriods\&quot;: [{\&quot;activeDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Wednesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;rounding\&quot;: \&quot;down\&quot;, \&quot;amountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;amountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentAdd\&quot;: \&quot;0.0\&quot;, \&quot;percentOff\&quot;: \&quot;0.0\&quot;, \&quot;localAmountAdd\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 50, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localAmountOff\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;level\&quot;: \&quot;string\&quot;, \&quot;inventoryTypes\&quot;: [{\&quot;code\&quot;: \&quot;DIT2\&quot;, \&quot;pacCode\&quot;: \&quot;AB\&quot;, \&quot;refInventoryType\&quot;: \&quot;RM1\&quot;, \&quot;inventoryTypes\&quot;: null, \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: \&quot;Flat\&quot;, \&quot;amountValue\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;roomClasses\&quot;: [{\&quot;code\&quot;: \&quot;string\&quot;, \&quot;pacCode\&quot;: null, \&quot;refInventoryType\&quot;: null, \&quot;inventoryTypes\&quot;: [\&quot;string\&quot;], \&quot;inventoryTypeRate\&quot;: null, \&quot;marketType\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;amountValue\&quot;: null, \&quot;percentValue\&quot;: null, \&quot;monetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 40, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;localMonetaryAmount\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 20, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;extraPersonCharge\&quot;: {\&quot;currencyCode\&quot;: null, \&quot;value\&quot;: 30, \&quot;guestViewableInd\&quot;: null, \&quot;numberOfDecimals\&quot;: 0}, \&quot;changedInd\&quot;: true, \&quot;guestsByType\&quot;: {\&quot;guestType\&quot;: \&quot;Adult\&quot;, \&quot;count\&quot;: 2}}], \&quot;occupancy\&quot;: 2, \&quot;guestTypeEnum\&quot;: \&quot;Adult\&quot;}], \&quot;priceGridStayRestriction\&quot;: {\&quot;restrictionTypeEnum\&quot;: null, \&quot;id\&quot;: null, \&quot;timespan\&quot;: null, \&quot;inventoryTypeCode\&quot;: null, \&quot;yieldClass\&quot;: null, \&quot;lengthOfStay\&quot;: [{\&quot;min\&quot;: \&quot;0\&quot;, \&quot;max\&quot;: \&quot;0\&quot;}], \&quot;statusEnum\&quot;: null, \&quot;requiredCheckInDowEnum\&quot;: null, \&quot;stayOverDowEnum\&quot;: null, \&quot;noStayDowEnum\&quot;: null, \&quot;closedToArrivalsDowEnum\&quot;: [\&quot;Monday\&quot;, \&quot;Tuesday\&quot;, \&quot;Friday\&quot;, \&quot;Saturday\&quot;, \&quot;Sunday\&quot;], \&quot;code\&quot;: null, \&quot;type\&quot;: null, \&quot;dowEnum\&quot;: null, \&quot;applyToCheckInInd\&quot;: null, \&quot;codeTypeEnum\&quot;: null, \&quot;roomBucket\&quot;: null, \&quot;businessUsageEnum\&quot;: null, \&quot;protections\&quot;: null}, \&quot;referencePriceGridCode\&quot;: \&quot;J4U\&quot;, \&quot;triggeringAction\&quot;: null, \&quot;pricing\&quot;: {\&quot;frequencyEnum\&quot;: \&quot;PerNight\&quot;, \&quot;methodsEnum\&quot;: \&quot;PerPerson\&quot;, \&quot;bookingPatternEnum\&quot;: \&quot;DayOfCheckIn\&quot;}, \&quot;taxDetails\&quot;: [{\&quot;id\&quot;: null, \&quot;code\&quot;: null, \&quot;typeEnum\&quot;: null, \&quot;name\&quot;: null, \&quot;includedInd\&quot;: true, \&quot;feeInd\&quot;: false, \&quot;dependencyAttribute\&quot;: null, \&quot;computationTypeEnum\&quot;: null, \&quot;computationModeEnum\&quot;: null, \&quot;chargeUnitEnum\&quot;: null, \&quot;chargeFrequencyEnum\&quot;: null, \&quot;maxChargeFrequency\&quot;: null, \&quot;chargeUnitExempt\&quot;: null, \&quot;paidAtHotelInd\&quot;: null, \&quot;exemptInd\&quot;: null, \&quot;relatedTaxIds\&quot;: null, \&quot;feeTaxableInd\&quot;: null, \&quot;description\&quot;: null, \&quot;longDescription\&quot;: null, \&quot;overrideInd\&quot;: null, \&quot;version\&quot;: null, \&quot;lastUpdate\&quot;: null, \&quot;modelVersion\&quot;: null, \&quot;activationStatusEnum\&quot;: null, \&quot;paymentAtHotelStatusEnum\&quot;: null, \&quot;chargePerPersonInd\&quot;: null, \&quot;residencyEnum\&quot;: null, \&quot;source\&quot;: null, \&quot;negotiableInd\&quot;: null, \&quot;relatedTaxCodes\&quot;: null, \&quot;scope\&quot;: null, \&quot;items\&quot;: null, \&quot;periods\&quot;: [{\&quot;timespan\&quot;: {\&quot;start\&quot;: \&quot;2024-02-19\&quot;, \&quot;end\&quot;: \&quot;2024-02-19\&quot;, \&quot;duration\&quot;: null}, \&quot;dowEnum\&quot;: null, \&quot;progressiveLOSInd\&quot;: null, \&quot;values\&quot;: null}], \&quot;reservationOverridableInd\&quot;: null, \&quot;overrideScopes\&quot;: null, \&quot;interfaceTaxName\&quot;: null, \&quot;baseRateAmountQuotedInd\&quot;: null, \&quot;excludedFromRedemptionRateInd\&quot;: null}], \&quot;classificationsEnum\&quot;: [\&quot;Addon\&quot;], \&quot;currencyCode\&quot;: \&quot;USD\&quot;}}&quot;<br>  )<br><br>  val updatedMergedPricePlanChunkByInvMessage = MergedPricePlan(<br><br>    PricePlan(<br><br>      propertyCode = PropertyCode(&quot;BOGAR&quot;), // Updated from &quot;BOGAR&quot;<br><br>      generationTime = &quot;2024-04-30T13:10:43.080Z&quot;, // Updated from &quot;2021-01-01&quot;<br><br>      pricePlanDetails = PricePlanDetails(<br><br>        code = &quot;DPP2&quot;, // Updated from &quot;P1&quot;<br><br>        actionEnum = Some(&quot;Create&quot;), // Updated from &quot;CREATE&quot;<br><br>        statusEnum = Some(&quot;Active&quot;), // Updated from &quot;ACTIVE&quot;<br><br>        priceGridDetails = Some(PriceGridDetails(<br><br>          name = Some(&quot;New price grid&quot;), // Updated from &quot;Standard&quot;<br><br>          typeEnum = Some(&quot;FlatRate&quot;), // Updated from &quot;Regular&quot;<br><br>          applicableToAllInventoryTypesInd = Some(true), // No change<br><br>          taxInclusionEnum = Some(&quot;IncludedInRate&quot;), // Updated from &quot;Inclusion 1&quot;<br><br>          productCategoryCode = Some(&quot;string&quot;), // Updated from &quot;Category 1&quot;<br><br>          extraPersonCharge = Some(CurrencyAmount(Some(&quot;USD&quot;), 30, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localExtraPersonCharge = Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          priceEndsWith = Some(&quot;30&quot;), // Updated from &quot;priceendswith&quot;<br><br>          productClassCode = Some(&quot;SPA&quot;), // Updated from &quot;classCode&quot;<br><br>          productCode = Some(&quot;MASSAGE&quot;), // Updated from &quot;pCode&quot;<br><br>          maxFreeNonAdults = Some(0), // Updated from 1<br><br>          freeGuestsByType = Seq(GuestsByType(Some(&quot;Teen&quot;), Some(2))) // Updated from Seq(GuestsByType(Some(&quot;guest&quot;), Some(1)))<br><br>        )),<br><br>        Some(YieldManagement(<br><br>          yieldChangedInd = Some(true), // Updated from false<br><br>          typeEnum = Some(&quot;AdjustmentValue&quot;), // Updated from &quot;type&quot;<br><br>          flatAmount = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localFlatAmount = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          percent = Some(&quot;0.0&quot;) // Updated from &quot;percentage&quot;<br><br>        )),<br><br>        Seq(RatePeriod(<br><br>          activeDowEnum = Seq(&quot;Monday&quot;, &quot;Wednesday&quot;, &quot;Friday&quot;, &quot;Saturday&quot;, &quot;Sunday&quot;), // Updated from Seq(&quot;Monday&quot;, &quot;Tuesday&quot;)<br><br>          timespan = Some(Timespan(Some(&quot;2024-02-19&quot;), Some(&quot;2024-02-19&quot;), None)), // Updated from Some(&quot;2025-07-20&quot;), Some(&quot;2025-08-10&quot;), Some(&quot;10&quot;)<br><br>          rounding = Some(&quot;down&quot;), // No change<br><br>          amountAdd = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          amountOff = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          percentAdd = Some(&quot;0.0&quot;), // Updated from &quot;10&quot;<br><br>          percentOff = Some(&quot;0.0&quot;), // Updated from &quot;10&quot;<br><br>          localAmountAdd = Some(CurrencyAmount(Some(&quot;USD&quot;), 50, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          localAmountOff = Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br><br>          level = Some(&quot;string&quot;), // Updated from &quot;level&quot;<br><br>          inventoryTypes = Seq(<br>            PricePoint(<br>              &quot;DIT2&quot;,<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 40, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(CurrencyAmount(Some(&quot;USD&quot;), 20, Some(false), Some(0))), // Updated from &quot;usd&quot;, 1234, Some(2)<br>              Some(true), // Updated from false<br>              Some(GuestsByType(Some(&quot;Adult&quot;), Some(2))) // Updated from GuestsByType(Some(&quot;guest&quot;), Some(1))<br>            )<br>          ),<br><br>          occupancy = Some(2), // Updated from 1<br><br>          guestTypeEnum = Some(&quot;Adult&quot;), // Updated from &quot;guestTypeEnum&quot;<br>          Some(&quot;USD&quot;)<br>        )),<br><br>        Some(StayRestriction(<br>          Seq(com.marriott.mgpapp.spark.priceplan.ingestion.entity.Range(Some(&quot;0&quot;), Some(&quot;0&quot;))), // Updated from Seq(Some(&quot;min&quot;), Some(&quot;max&quot;))<br>          Seq(&quot;string&quot;),<br>          Seq(&quot;String&quot;),<br>          Seq(&quot;String&quot;),<br>          Seq(&quot;String&quot;)<br>        )),<br><br>        referencePriceGridCode = Some(&quot;J4U&quot;), // Updated from &quot;refPriceGridCode&quot;<br><br>        pricing = Some(Pricing(<br>          Some(&quot;PerNight&quot;), // Updated from Some(&quot;freq&quot;)<br>          Some(&quot;PerPerson&quot;), // Updated from Some(&quot;methods&quot;)<br>          Some(&quot;DayOfCheckIn&quot;) // Updated from Some(&quot;book&quot;)<br>        )),<br><br>        classificationsEnum = Seq(&quot;Addon&quot;), // Updated from Seq(&quot;Classification 1&quot;, &quot;Classification 2&quot;)<br><br>        currencyCode = &quot;USD&quot; // No change<br><br>      )<br>    ), &quot;&quot;)<br><br><br>}</pre><p>HCL Interview:</p><p>— — — — — — — — — — — — -question1 — — — — — — — — — — — — — — — — — — — —</p><p>Consider you have a list [2,4,1,7,9] now you have to return the pair whose sum is 3 .</p><p>— — — — — — — — — — — Scala — — — — — — — — — — — — — — — — — — — — — —</p><p>Ist approach</p><pre>object Hello_world {<br>  <br>  def main(args:Array[String]) : Unit = {<br>    val lst = List(2,1,3,4,7)<br>    val pairs = for {<br>      i &lt;- lst<br>      j &lt;- lst<br>      if i+j == 3<br>      } yield (i,j)<br>      println(pairs)<br>  }</pre><p>2nd Approach</p><pre>  object Hello_world2 {<br>  def main(args:Array[String]):Unit = {<br>    <br>    val lst = List(2,3,4,5,1)<br>    val pairs = lst.flatMap(i=&gt; lst.map(j =&gt; (i,j) ) ).<br>    filter{ case (i,j) =&gt; i+j  == 3}<br>    println(pairs)<br>  }<br>  <br>  <br>}</pre><p>some notes for the 2nd approach:</p><ol><li><strong>Why Not </strong><strong>map((i, j) =&gt; ...)?</strong></li></ol><blockquote>You <strong>cannot directly</strong> write .map((i, j) =&gt; ...) because</blockquote><blockquote><strong>map operates on single elements</strong>, not pairs of elements. So map expects a function that operates on one element at a time.</blockquote><blockquote>lst.map(...) means you&#39;re applying a function to <strong>one element</strong> at a time, not pairs (i, j). That is why i =&gt; lst.map(j =&gt; (i, j)) is necessary.</blockquote><p><strong>2. Why the Nested Structure?</strong></p><blockquote>The structure i =&gt; lst.map(j =&gt; (i, j)) works because:</blockquote><blockquote><strong>Outer part</strong>: flatMap(i =&gt; ...) takes each element i from the original list.</blockquote><blockquote><strong>Inner part</strong>: For each i, you use lst.map(j =&gt; (i, j)) to create pairs where i is paired with every element j in the list.</blockquote><h3>Conceptual View:</h3><p>Let’s look at the nested structure with an example.</p><blockquote>For lst = List(2, 4, 1), if we break down the steps:</blockquote><blockquote><strong>flatMap</strong>: For each i in lst (e.g., first i = 2, then i = 4, etc.).</blockquote><blockquote><strong>map inside flatMap</strong>: For each i, you map it to a list of pairs (i, j), where j is every element of lst.</blockquote><blockquote>When i = 2: List((2, 2), (2, 4), (2, 1))</blockquote><blockquote>When i = 4: List((4, 2), (4, 4), (4, 1))</blockquote><blockquote>When i = 1: List((1, 2), (1, 4), (1, 1))</blockquote><blockquote>flatMap then flattens all these lists into a single list of pairs.</blockquote><h3>Why Not map((i, j) =&gt; ...) Directly?</h3><blockquote>map is designed to operate on single elements, not multiple elements like (i, j). Since lst contains individual elements like 2, 4, 1, not pairs, you cannot use .map((i, j) =&gt; ...) unless you are already working with pairs.</blockquote><p>object HelloWorld {<br> def main(args: Array[String]): Unit = {<br> val x= List(1,2,3,4,5,6,-1,-2,-3,-4)<br> val y = x.filter(num =&gt; num &gt; 0)<br> println(y)<br> }<br>}</p><p>when we try to reassign the list it was not possible because list are immutable</p><p>we can assign the list keeing the variable name same.</p><pre>reverse_tuple = result.map(lambda x:(x[1],x[0]))..<br>is its tuple enclosed it in the bracket</pre><pre>python</pre><pre>final_mapped = words.map(lambda x: (x[1].lower(), x[0]))</pre><pre>scala</pre><pre>val finalMapped = words.map { case (x1, x2) =&gt; (x2.toLowerCase, x1) }</pre><p>python:</p><p>scala:</p><p>For example, if you have a map operation that needs to swap the two elements of a tuple:</p><pre>scala</pre><pre>Copy code</pre><pre>val finalMapped = words.map { case (x1, x2) =&gt; (x2, x1) }</pre><p>If you want to use tuple accessors _1 and _2, it would be:</p><pre>scala</pre><pre>val finalMapped = words.map { case (x1, x2) =&gt; (x2, x1) }</pre><pre>val finalMapped = words.map(x =&gt; (x._2, x._1))</pre><pre>python<br></pre><pre>finalCount = wordCount.reduceByKey(lambda x,y :x+y).map(lambda x:(x[1],x[0]))</pre><h3>What is a SOAP API, and how does it differ from a REST API?</h3><p><strong>Answer:</strong></p><ul><li><strong>SOAP (Simple Object Access Protocol)</strong> is a protocol used for exchanging structured information in web services. SOAP APIs are more rigid than REST APIs, requiring strict adherence to XML standards and predefined rules.</li><li><strong>REST APIs</strong> are more flexible, often use JSON (but can also use XML), and operate with less overhead, making them more popular in modern web development.</li><li>SOAP is more suited for complex, enterprise-level services requiring strict contracts and security, while REST is widely used in simpler, more scalable web applications.</li></ul><p>When choosing between <strong>client</strong> and <strong>cluster</strong> deploy modes in Spark, the best choice depends on the use case, resources, and specific environment requirements. Here’s a comparison of the two modes:</p><h3>1. Client Mode:</h3><p>In <strong>client</strong> mode, the driver (the Spark application’s coordinator) runs on the machine where the Spark job was submitted (i.e., the client machine), while the executors run on the worker nodes in the cluster.</p><ul><li><strong>Advantages</strong>:</li><li><strong>Interactive Development</strong>: It is ideal for interactive applications such as notebooks (e.g., Jupyter, Zeppelin) where the driver needs to be close to the user for fast interaction.</li><li><strong>Debugging</strong>: Easier for debugging since the driver runs on the local machine, and logs are more easily accessible.</li><li><strong>Low-latency applications</strong>: Good for applications that require low-latency communication between the user and the driver (e.g., exploratory data analysis).</li><li><strong>Disadvantages</strong>:</li><li><strong>Resource limitation</strong>: The driver running on the client machine may have limited resources (CPU, memory) compared to cluster nodes.</li><li><strong>Network bottleneck</strong>: The client machine must be always connected to the cluster network, as all driver-related communication passes through it.</li><li><strong>Risk of failure</strong>: If the client machine disconnects or fails, the job can be disrupted.</li></ul><h3>2. Cluster Mode:</h3><p>In <strong>cluster</strong> mode, both the driver and executors run on the cluster worker nodes. The job submission client only submits the job and doesn’t stay connected after submission.</p><ul><li><strong>Advantages</strong>:</li><li><strong>Better Resource Utilization</strong>: The driver is run on a worker node with more resources (memory, CPU) than on a client machine.</li><li><strong>Scalability</strong>: It is better suited for large-scale, long-running production jobs, as it doesn’t depend on the client machine for execution.</li><li><strong>Resiliency</strong>: More resilient since the client’s failure doesn’t impact the job running on the cluster. The job runs independently of the submitting machine.</li><li><strong>Best for Production</strong>: Ideal for non-interactive, batch jobs and production workloads where job isolation and stability are important.</li><li><strong>Disadvantages</strong>:</li><li><strong>More challenging to debug</strong>: Logs are scattered across different cluster nodes (driver and executors) and need to be accessed on the cluster.</li><li><strong>Less interactive</strong>: Not ideal for interactive sessions as the driver isn’t on the local machine.</li></ul><h3>When to Use Which Mode:</h3><ul><li><strong>Client Mode</strong>:</li><li>Ideal for <strong>development</strong>, <strong>testing</strong>, and <strong>interactive</strong> applications (e.g., notebooks or local testing).</li><li>Useful if the Spark application needs to run interactively on your machine.</li><li>When the driver has low memory and CPU requirements.</li><li><strong>Cluster Mode</strong>:</li><li>Best suited for <strong>production environments</strong> and <strong>batch jobs</strong> where the application runs independently of the client.</li><li>Necessary for <strong>large-scale jobs</strong> that require significant driver resources, which the client machine may not provide.</li><li>Ideal when you need reliability and can’t depend on the client machine to remain connected.</li></ul><h3>Conclusion:</h3><ul><li><strong>Client mode</strong> is better for <strong>development, debugging, or interactive workloads</strong> where ease of access to logs and real-time results is critical.</li><li><strong>Cluster mode</strong> is better for <strong>large-scale, production-grade jobs</strong> that require distributed execution and job resiliency.</li></ul><p>For most production scenarios, <strong>cluster mode</strong> is the preferred choice, as it provides scalability and does not depend on the client machine’s stability or resources.</p><p>week 19:</p><p><a href="https://chatgpt.com/c/474e0ee8-a5d2-43f7-80a2-209b5b7a0ddd">https://chatgpt.com/c/474e0ee8-a5d2-43f7-80a2-209b5b7a0ddd</a></p><h3>Data Modeling</h3><pre><br><br>A way to structure your data so that it fits your needs in the best possible way.<br><br>Needs can be different based on what system are we Modeling &amp; who is the end user?<br><br>model the table in a way that reduces the storage space, <br>modeling it in a way that queries run faster, <br>modeling in a way that user can easily query<br>etc..<br><br>In a transactional system (OLTP) we generally use Normalization -<br>It&#39;s a technique to divide one big table into multiple smaller tables with an intent to reduce the redundancy.<br><br>However, OLTP systems are not meant to do reporting?<br>our reporting work can overload the OLTP systems<br><br>Datawarehouse (DWH) is best fit for reporting purpose (OLAP)<br><br>What is DWH - it&#39;s like a Database but the objective is to make your analytical queries faster.<br><br>Dimensional Modeling is one of the well know techniques for modeling a DWH.<br><br>here are 2 definitions that you should know - <br><br>- &quot;Dimensional Modeling is a design technique for Databases intended to support end user queries in a DWH&quot;<br>Ralph Kimball<br><br> - the process of modeling a business process into a series of facts and dimension tables designed for analysis.<br><br>Key highlights of a Transactional DB design<br>- designed towards fast maintenance of data<br>- inserting and updating is quick<br>- very small sets of data is retrieved in a query<br>- Data consistency is critical<br>- Focus is on customers who are entering the data<br><br>Reporting DB design<br>- copy of transactional data (not exactly the same way)<br>- the resulting model reflects the kind of questions business wants to ask rather than the functions of underlying operational system.<br>- Descriptive data like customer name, customer address is separated from the quantity data such as order quantity, order amount.<br>- larger datsets<br>- insert and update speed is not relevant<br>- performance focus is on retrieving the data quickly.<br><br>fun facts regarding facts &amp; dimension tables<br>=&gt; dimension tables would contain more than 90% of total columns, however data in fact table would be more as we keep getting new data.<br>=&gt; in fact table the foreign keys might take more space then the actual fact information .<br><br>When storage was costly, Dimension modeling ruled and it&#39;s even ruling today.<br><br>Now a days OBT (one big table) is also becoming popular, because storage space is not a big concern. We can pay little extra for storage if that helps us to build faster queries, and it&#39;s easy for end users to query it.<br><br><br><br></pre><h3>Normalization vs Denormalization !</h3><pre><br><br><br>Normalization is a process of dividing the data into multiple smaller tables with an intent to reduce data redundancy &amp; inconsistency. <br><br>However, Denormalization is totally opposite of above idea. <br><br>Denormalization is the technique of combining the data into a big single table. This definitely leads to redundancy in the data.<br><br>Note: Redundancy causes inconsistency -<br>Consider that same data is repeated 2 times and when updating you update at one place and forget to do at second place. This leads to inconsistent state.<br><br>When retrieving data from Normalized tables we need to read many tables and perform join which is a costly operation.<br>However, when reading the data from denormalized tables it&#39;s quite fast as no joins are required.<br><br>When to use Normalized tables vs Denormalized ones?<br>when we talk about OLTP systems (Online Transaction Processing) where we deal with lot of insert, delete and updates then you should go for Normalized tables.<br><br>However, when you talk about OLAP (Online Analytical Processing) systems where you need to analyse historical data then Denormalized tables are best fit. Since you wont be doing updates on data here, even though after having redundancy we wont end up in inconsistent state.<br><br>Let&#39;s take a simple Example - when you make purchase on amazon then it requires a OLTP system (a rdbms kind of database). Here the tables should be normalized.<br><br>when amazon is doing data analysis of historical data, they will create denormalized tables just to make sure analysis is faster and costly joins can be avoided.</pre><h3>Spark configuration of executors:</h3><p>better explained in this article: <a href="https://medium.com/@ankitjuneja/spark-performance-optimizations-46204d99516f">https://medium.com/@ankitjuneja/spark-performance-optimizations-46204d99516f</a></p><pre>Lets say you have a 20 node spark cluster<br><br>Each node is of size - 16 cpu cores / 64 gb RAM<br><br>Let&#39;s say each node has 3 executors,<br>with each executor of size - 5 cpu cores / 21 GB RAM<br><br>=&gt; 1. What&#39;s the total capacity of cluster?<br><br>We will have 20 * 3 = 60 executors<br>Total CPU capacity: 60 * 5 = 300 cpu Cores<br>Total Memory capacity: 60 * 21 = 1260 GB RAM<br><br>=&gt; 2. How many parallel tasks can run on this cluster?<br><br>We have 300 CPU cores, we can run 300 parallel tasks on this cluster.<br><br>=&gt; 3. Let&#39;s say you requested for 4 executors then how many parallel tasks can run?<br><br>so the capacity we got is 20 cpu cores<br>84 GB RAM<br>so a total of 20 parallel tasks can run.<br><br>=&gt; 4. Let&#39;s say we read a csv file of 10.1 GB stored in datalake and have to do some filtering of data, how many tasks will run?<br><br>if we create a dataframe out of 10.1 GB file we will get 81 partitions in our dataframe. (will cover in my next post on how many partitions are created)<br><br>so we have 81 partitions each of size 128 mb, the last partition will be a bit smaller.<br><br>so our job will have 81 total tasks.<br>but we have 20 cpu cores<br><br>lets say each task takes around 10 second to process 128 mb data.<br>so first 20 tasks run in parallel,<br>once these 20 tasks are done the other 20 tasks are executed and so on...<br><br>so totally 5 cycles, if we think the most ideal scenario.<br><br>10 sec + 10 sec + 10 sec + 10 sec + 8 sec<br><br>first 4 cycles is to process 80 tasks all of 128 mb,<br>last 8 sec is to process just one task of around 100 mb, so it takes little lesser but 19 cpu cores were free during this time.<br><br>=&gt; 5. is there a possibility of, out of memory error in the above scenario?<br><br>Each executor has 5 cpu cores and 21 gb ram.<br>This 21 gb RAM is divided in various parts -<br>300 mb reserved memory,<br>40% user memory to store user defined variables/data. example hashmap<br>60% spark memory - this is divided 50:50 between storage memory and execution memory.<br><br>so basically we are looking at execution memory and it will be around 28% roughly of the total memory allotted.<br><br>so consider around 6 GB of 21 GB memory is meant for execution memory.<br>per cpu core we have (6 GB / 5 cores) = 1.2 GB execution memory.<br>That means our task can roughly handle around 1.2 GB of data.<br>however, we are handling 128 mb so we are well under this range</pre><h3>Best example of optimisation for pipeline:</h3><pre>2 days ago<br><br>Lets say you have 5 TB of data stored in your Amazon S3 bucket.<br><br>Lets say this data has 500 million records, 100 columns.<br><br>Suppose there are 100 cities and you want to get the data for a particular city, and you want to retrieve only 10 columns.<br><br>~ considering each city has equal amount of records, <br>we want to get 1% of data in terms of number of rows<br>and 10% in terms of columns<br><br>thats roughly 0.1% of the actual data which might be 5 GB roughly.<br><br>Now lets the pricing if you are using serverless technology like AWS Athena <br><br>- the worst case you end up having the data in a csv format (row based) with no compression. you end up scanning the entire 5 TB data and you pay $25 for this query. (The charges are $5 for each TB of data scanned)<br><br>Now lets try to improve it..<br><br>- use a columnar file format like parquet with snappy compression which takes lesser space so your 5 TB data might roughly become 2 TB (actually it will be even lesser)<br><br>- partition this based on city so that we have 1 folder for each city.<br><br>This way you have 2 TB data sitting across 100 folders, but you have to scan just one folder which is 20 GB,<br><br>Not just this you need 10 columns out of 100 so roughly you scan 10% of 20 GB (as we are using columnar file format)<br><br>This comes out to be 2 GB only.<br><br>so how much do we pay?<br>just $.01 which is 2500 times lesser than what you paid earlier.<br><br>This is how you save cost.<br><br>what we did?<br><br>- using columnar file formats for column pruning<br>- using partitioning for row pruning<br>- using efficient compression techniques</pre><h3>𝐖𝐡𝐚𝐭 𝐚𝐫𝐞 𝐬𝐨𝐦𝐞 𝐨𝐩𝐭𝐢𝐦𝐢𝐳𝐚𝐭𝐢𝐨𝐧 𝐭𝐞𝐜𝐡𝐧𝐢𝐪𝐮𝐞𝐬 𝐢𝐧 𝐒𝐩𝐚𝐫𝐤?</h3><pre><br>Here are some common 𝐨𝐩𝐭𝐢𝐦𝐢𝐳𝐚𝐭𝐢𝐨𝐧 techniques:<br><br>Data Serialization: Use efficient data formats like Parquet or Avro, which are columnar and support compression, reducing both storage and I/O costs.<br><br>Caching: Use persist() or cache() to store frequently accessed data in memory, reducing the need to recompute it.<br><br>Broadcast Variables: For small datasets, broadcasting ensures that data is sent to all worker nodes, minimizing communication overhead.<br><br>Partitioning: Properly partition data based on keys can improve parallel processing. Use repartition() or coalesce() to adjust the number of partitions.<br>Avoiding Shuffles: Minimize operations that trigger shuffles (like groupBy or join) by rethinking your data transformations and using mapPartitions() when possible.<br><br>Tuning Spark Configurations: Adjust Spark configurations like executor memory, number of cores, and parallelism based on your cluster&#39;s resources.<br><br>Predicate Pushdown: Filter data as early as possible in the data processing pipeline to reduce the amount of data shuffled across the network.</pre><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=268464dc0aed" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[AWS DATA ENINEER]]></title>
            <link>https://medium.com/@ankitjuneja/aws-data-enineer-3d7e8dbc7978?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/3d7e8dbc7978</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Mon, 29 Jan 2024 06:27:21 GMT</pubDate>
            <atom:updated>2024-01-30T09:57:13.055Z</atom:updated>
            <content:encoded><![CDATA[<h3>DATALAKE FORMATION ARCITECTURE:</h3><blockquote>data will come from te sql, nosql and streamin data ten te raw data will be processed and ten curated witin te lue jobs and in te final it will be dumped into datawareouse.</blockquote><figure><img alt="" src="https://cdn-images-1.medium.com/max/650/1*cTvMW91h6haMaR4WP7gGZQ.png" /></figure><h3>create a new IAM user lakeformation-admin and rant te full policy permission</h3><h4>te two permissions are</h4><blockquote>administrator access</blockquote><blockquote>aws lakeformationdataAdmin</blockquote><h3><strong>o to aws datalake formation and add te iam user(lake-formation-admin )into it</strong></h3><blockquote>now rant access to te iam user to create te database</blockquote><h3>o to s3 , create a bucket demo-lakeformation-raw..&gt; enabliin versionin and encryptuion</h3><blockquote>crreate tree folders input,output,atena into te bucket</blockquote><p>now buckets are created , datalake is created….next step is to make te database in datlake and link s3 buckets wit databases.</p><h3>CREATE databases in datlake and connect tem wit s3 buckets.</h3><figure><img alt="" src="https://cdn-images-1.medium.com/max/900/1*W0kcOMBOxM9Ozcf6GdGo1w.png" /></figure><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=3d7e8dbc7978" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[CREDIT CARD FRAUD DETECTION.]]></title>
            <link>https://medium.com/@ankitjuneja/credit-card-fraud-detection-89719d51adfc?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/89719d51adfc</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Sun, 21 Jan 2024 15:32:32 GMT</pubDate>
            <atom:updated>2024-01-21T15:32:32.769Z</atom:updated>
            <content:encoded><![CDATA[<h4>Task 1:</h4><p>Copy “card_transactions.csv” file from local system to HDFS.</p><h4>Table creation tasks :</h4><h4>Task 2:</h4><p>Create the “card_transactions” table in MySQL based on the card_transactions.csv file structure.</p><h4>Task 3:</h4><p>Do a sqoop export to the database for card_transactions.csv and delete the file from HDFS.</p><h4>Task 4:</h4><p>On “member_score” and “member_details” create a normal hive external table.</p><h4>Task 5:</h4><p>Create a special “card_transactions” Hbase table managed by Hive.</p><h4>Task 6:</h4><p>Create a Hbase “lookup” table with columns — member_id, card_id, UCL, timestamp, zipcode, credit_score.</p><h4>Batch Processing tasks:</h4><h4>Task 7:</h4><p>Sqoop import member_score from AWS-RDS to Hive. (Full load import, has to be refreshed every week)</p><h4>Task 8:</h4><p>Sqoop import member_details from AWS-RDS to Hive. (Incremental load import in append mode based on member_id for every 8hrs)</p><h4>Task 9:</h4><p>Sqoop import card_transactions to HDFS from MySQL. (This is a one-time full load activity. The card_transactions table will be updated with new transactions while in streaming mode.)</p><h4>Scheduling tasks:</h4><h4>Task 10:</h4><p>Schedule a sqoop import job using Airflow to import member_score from AWS-RDS to Hive on a full-load.</p><h4>Task 11:</h4><p>Schedule a sqoop import job using Airflow to import member_details from AWS-RDS to Hive on an incremental append mode for every 8hrs.</p><h4>Integration tasks:</h4><h4>Task 12:</h4><p>Spark-HBase Integration</p><ol><li>For populating the card_transactions table.</li><li>For populating the look_up table.</li></ol><h4>Task 13:</h4><p>Spark-Hive Integration for spark stream processing.</p><h4>Task 14:</h4><p>Access the hive tables using apache spark and calculate the UCL.</p><h4>Streaming tasks:</h4><h4>Task 15:</h4><p>Producer to create the transactions in JSON format, to be added and queued in Kafka topics.</p><h4>Task 16:</h4><p>Spark structured streaming program as a Consumer that will consume the data from kafka topics.</p><p>Task 17:</p><p>Retrieve the timestamp and zipcode of the last transaction of each card.</p><h4>Task 18:</h4><p>Processing in Spark Streaming -</p><p>Task 19.1 : Validating RULE 1 -&gt; “credit_score &gt; 200”</p><p>Task 19.2 : Validating RULE 2 -&gt; “transaction amount &lt;= UCL”</p><p>Task 19.3 : Validating RULE 3 -&gt; “zipcode distance within threshold”</p><h4>Task 19:</h4><p>Based on the above rules, the entire transaction along with status should be updated in the card_transactions table.</p><h4>Task 20:</h4><p>Schedule a job for validating rules by comparing the incoming data from the POS terminals in JSON format with the values in the lookup table.</p><h4>Task 21:</h4><p>If the transaction was marked genuine, then we need to update the lookup table with the new timestamp and the zipcode.</p><h4>Task 22:</h4><p>Schedule a job for populating the lookup table.</p><p>ARCITECTURE</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*92Oq4S1XZzHatU0kccKVBw.png" /></figure><p>DETAILED ARCITECTURE:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*YRgiGmZQnu_FJ5Dc8SNRVg.png" /></figure><h3><strong>data we are ettin in tis is tere will be total 4 files 3 csv and i json(streamin data)</strong></h3><h3>We will be considering the following tables :</h3><p><strong>card_member(RDS mysql table)-&gt;</strong></p><p>This table consists of cardholder data and the data is added into this table by a third party service. Following are the fields in the card_member table.</p><ul><li>card_id (Card number)</li><li>member_id (Member id of the cardholder)</li><li>member_joining_dt (Joining data and time of a new member)</li><li>card_purchase_dt (Purchase date of the card)</li><li>country (Name of the country where the card was purchased)</li><li>city (Name of the city where the card was purchased)</li></ul><p><strong>card_transactions</strong> -&gt; This table consists of all the transactions carried out by swiping at the POS system. Following are the fields in the card_transactions table.</p><ul><li>card_id (Card number)</li><li>member_id (Member id of the cardholder)</li><li>amount (Amount swiped with the particular card_id)</li><li>postcode (marks the location of the event based on the zipcode at which the card was swiped)</li><li>pos_id (POS terminal ID using which the card was swiped)</li><li>transaction_dt (date and time of transaction)</li><li>Status (Genuine / Fraud value based on whether the transaction was approved or declined)</li></ul><p><strong>member_score(rds mysql table)-&gt;</strong> This table consists of all the credit scores of the card holder. Following are the fields in the member_score table.</p><ul><li>member_id (Member id of the cardholder)</li><li>score (The score assigned to a member defining his/her credit history)</li></ul><p>Apart from the above data, There will be real-time data that will be streaming in, generated by the</p><p><strong>POS(Point of Sale) systems. This data will be in JSON format as shown below :</strong></p><p>{</p><p>“card_id” : 487654323445689,</p><p>“member_id” : 000987654123456,</p><p>“amount” : 245600,</p><p>“pos_id” : 78765324158934,</p><p>“postcode” : 33946,</p><p>“transaction_dt” : 09–01–2021 18:00:00</p><p>}</p><p><strong>STEP1: SET UP AMAZON RDS TABLES AND PUT THE DATA INTO BOTH THE TABLES USING SQOOP EXPORT:</strong></p><blockquote><strong>connect to the RDS DB Instance using the following :</strong></blockquote><blockquote><strong>mysql -h &lt;Endpoint&gt; -P 3306 -u &lt;Master Username&gt; -p</strong></blockquote><blockquote><strong>Once connected,</strong></blockquote><blockquote><strong>1.create the card_member and member_score tables</strong></blockquote><blockquote><strong>2.sqoop export​ the data from HDFS to the tables.</strong></blockquote><h3><strong>for the project purpose we have taken the data in csv format and then exporting it .</strong></h3><figure><img alt="" src="https://cdn-images-1.medium.com/max/654/1*X2rJlPbJbiajJv-T20u2Xw.png" /></figure><h3>task2:</h3><figure><img alt="" src="https://cdn-images-1.medium.com/max/564/1*dIEZcEkwyBHg9uk7ifMhoA.png" /></figure><p>task 3:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/669/1*24I_SiSYxHVgB11vxhvr3g.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/623/1*PRcTYxBq32M668PPNCcZNQ.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/517/1*MsigQsanjeqDJ7rInpw8mg.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/520/1*UoxmBY1WzNafA79MAU6W1g.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*0Ba59Jn6mgDUN_phgXjRuQ.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/586/1*NsijIMHpWcpk6Cen8XeuSA.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/611/1*HotJRTGAt4MF1YT_V1c20g.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/514/1*j72BBIRM66jRTboMm-vStg.png" /></figure><p>task 4:</p><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=89719d51adfc" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[Scala]]></title>
            <link>https://medium.com/@ankitjuneja/scala-9fb6eeb8edc4?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/9fb6eeb8edc4</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Wed, 17 Jan 2024 08:42:24 GMT</pubDate>
            <atom:updated>2024-05-19T05:49:48.383Z</atom:updated>
            <content:encoded><![CDATA[<p>Scala ist code wordcount</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*zR7chdSkDCIIimPb_U40oQ@2x.jpeg" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*O3i2mxOsxHcwOtrTWwBDEw@2x.jpeg" /></figure><p>App me already main method hota hai</p><pre>import org.apache.log4j.{Level, Logger}<br>import org.apache.spark.{SparkConf, SparkContext}<br><br>object WordCount extends App {<br>  Logger.getLogger(&quot;org&quot;).setLevel(Level.ERROR)<br><br>  val conf = new SparkConf().setAppName(&quot;WordCount&quot;).setMaster(&quot;local[*]&quot;)<br>  val sc = new SparkContext(conf)<br><br>  val input = sc.textFile(&quot;D:/2023/trendy_certi/week9/unlocked/search.txt&quot;)<br>  val words = input.flatMap(x =&gt; x.split(&quot; &quot;))<br>  val wordCount = words.map(x =&gt; (x, 1))<br>  val finalCount = wordCount.reduceByKey((a, b) =&gt; a + b)<br><br>  finalCount.collect.foreach(println)<br>}</pre><p>or</p><blockquote><strong>asd input.flatMap(_.split(“ “))</strong></blockquote><blockquote><strong>.map(_.toLowerCase())</strong></blockquote><blockquote><strong>.map((_,1))</strong></blockquote><blockquote><strong>.reduceByKey(_+_)</strong></blockquote><blockquote><strong>.collect.foreach(println)</strong></blockquote><pre>import org.apache.log4j.Level<br>import org.apache.log4j.Logger<br>import org.apache.spark.SparkConf<br>import org.apache.spark.SparkContext<br><br><br>  //after spark4<br>  <br>  <br>  object wordcount5 extends App {<br>  Logger.getLogger(&quot;org&quot;).setLevel(Level.ERROR)<br>  val sc = new SparkContext (&quot;local[*]&quot;,&quot;wordcount&quot;)<br>  val input = sc.textFile(&quot;D:/2023/trendy_certi/week9/unlocked/search.txt&quot;)<br>  //in one go<br>  input.flatMap(_.split(&quot; &quot;)).map(_.toLowerCase()).map((_,1)).reduceByKey(_+_).collect.foreach(println)<br>  <br>  <br>  <br>}</pre><p>to hold on to te screen to see te dag …write tis line in te end as we dont ave history server set.</p><h3>scala.io.StdIn.readLine()</h3><h3>foreach</h3><blockquote>for (Result &lt;- Results)</blockquote><blockquote>{</blockquote><blockquote>val word = Result._1</blockquote><blockquote>val count = Result._2</blockquote><blockquote>println(s”$word : $count”)</blockquote><blockquote>}</blockquote><h3>sortbyKey</h3><blockquote>requires the reversing of the tuple</blockquote><blockquote>val reverseTuple = finalCount.map(x=&gt;(x._2,x._1))</blockquote><blockquote>val sortedResults = reverseTuple.sortByKey(false).map(x=&gt;(x._2,x._1))</blockquote><blockquote>sortedResults.collect.foreach(println)</blockquote><h3>sortBy</h3><blockquote>no reversin is required</blockquote><blockquote>no true false is required</blockquote><blockquote>val sortedResults = finalCount.sortBy(x=&gt;x._2)</blockquote><blockquote>sortedResults.collect.foreach(println)</blockquote><p>}</p><p>New Sparkcontext</p><p>Also split ka use</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*HBc4W1rr0BDDbC3MXYN9GA@2x.jpeg" /></figure><pre><br>import org.apache.log4j.Level<br>import org.apache.log4j.Logger<br>import org.apache.spark.SparkConf<br>import org.apache.spark.SparkContext<br><br><br><br>  <br>  <br>  object movie2 extends App {<br>  Logger.getLogger(&quot;org&quot;).setLevel(Level.ERROR)<br>  val sc = new SparkContext (&quot;local[*]&quot;,&quot;wordcount&quot;)<br>  val input = sc.textFile(&quot;D:/2023/trendy_certi/week9/unlocked/movie.data&quot;)<br>  val mappedInput = input.map(x=&gt;x.split(&quot;\t&quot;)(2))<br>  val ratings = mappedInput.map(x =&gt;(x,1))<br>  val reducedRatings = ratings.reduceByKey((x,y)=&gt; x+y)<br>  val results = reducedRatings.collect<br>  results.foreach(println)</pre><p>one thing is forsure in scala we can use function directly in bracket instead we need to use it like .length,.mkString,.toList</p><p>sample program:</p><blockquote><strong>This IS A wonDerFUl dAY display and count the capital letters in this in python</strong></blockquote><h3>python</h3><pre>text = &quot;This IS A wonDerFUl dAY&quot;<br>capital_letters = [char for char in text if char.isupper()]<br><br># Display the capital letters<br>print(&quot;Capital letters:&quot;, &#39;&#39;.join(capital_letters))<br><br># Count the capital letters<br>count_capital_letters = len(capital_letters)<br>print(&quot;Number of capital letters:&quot;, count_capital_letters)</pre><h3>scala1:</h3><pre>object Main extends App {<br>  val text = &quot;This IS A wonDerFUl dAY&quot;<br>  <br>  // Filter and collect capital letters<br>  val capitalLetters = text.filter(_.isUpper).toList<br>  <br>  // Display the list of capital letters<br>  println(&quot;List of capital letters: &quot; + capitalLetters.mkString)<br><br>  // Count the capital letters<br>  val countCapitalLetters = capitalLetters.length<br>  println(&quot;Number of capital letters: &quot; + countCapitalLetters)<br>}</pre><h3>scala2:</h3><pre>import org.apache.spark.{SparkConf, SparkContext}<br><br>object CapitalLetters {<br>  def main(args: Array[String]): Unit = {<br>    // Create a SparkConf and SparkContext<br>    val conf = new SparkConf().setAppName(&quot;CapitalLetters&quot;)<br>    val sc = new SparkContext(conf)<br><br>    // Define the text<br>    val text = &quot;This IS A wonDerFUl dAY&quot;<br><br>    // Convert the text to an RDD<br>    val textRDD = sc.parallelize(text.toCharArray)<br><br>    // Filter and collect capital letters<br>    val capitalLetters = textRDD.filter(char =&gt; char.isUpper).collect()<br><br>    // Display the list of capital letters<br>    println(&quot;List of capital letters: &quot; + capitalLetters.mkString)<br><br>    // Count the capital letters<br>    val countCapitalLetters = capitalLetters.length<br>    println(&quot;Number of capital letters: &quot; + countCapitalLetters)<br><br>    // Stop the SparkContext<br>    sc.stop()<br>  }<br>}</pre><p>If you want to check each word in a sentence for capital letters and process them separately, you’ll need to split the sentence into words and then apply the same filtering logic</p><pre>object Main extends App {<br>  val text = &quot;This IS A wonDerFUl dAY&quot;<br>  <br>  // Split the text into words<br>  val words = text.split(&quot; &quot;)<br>  <br>  // Process each word to filter and collect capital letters<br>  words.foreach { word =&gt;<br>    // Filter and collect capital letters in the word<br>    val capitalLetters = word.filter(_.isUpper).toList<br>    <br>    // Display the list of capital letters for each word<br>    println(s&quot;List of capital letters in word &#39;$word&#39;: &quot; + capitalLetters.mkString)<br>    <br>    // Count the capital letters in the word<br>    val countCapitalLetters = capitalLetters.length<br>    println(s&quot;Number of capital letters in word &#39;$word&#39;: &quot; + countCapitalLetters)<br>  }<br>}</pre><h3>pyspark:</h3><pre>from pyspark.sql import SparkSession<br><br># Create a Spark session<br>spark = SparkSession.builder.appName(&quot;CapitalLetters&quot;).getOrCreate()<br><br># Define the text<br>text = &quot;This IS A wonDerFUl dAY&quot;<br><br># Convert the text to an RDD (Resilient Distributed Dataset)<br>text_rdd = spark.sparkContext.parallelize(list(text))<br><br># Filter and collect capital letters<br>capital_letters = text_rdd.filter(lambda char: char.isupper()).collect()<br><br># Display the list of capital letters<br>print(&quot;List of capital letters:&quot;, &#39;&#39;.join(capital_letters))<br><br># Count the capital letters<br>count_capital_letters = len(capital_letters)<br>print(&quot;Number of capital letters:&quot;, count_capital_letters)<br><br># Stop the Spark session<br>spark.stop()</pre><p>First basic diff.</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/467/0*fQ6DwyICNcpHuWeE.jpeg" /></figure><ol><li>import org.apachr.spark.SparkContext</li><li>object wordcount extends App</li><li>x=&gt; x.split(“, “)(0).tofloat</li><li>rdd2 = rdd1.collect</li><li>rdd2.foreach(println)</li><li>for(a&lt;-result)</li><li>,map(x=&gt;(x_2,x_1)</li><li>.sortByKey(ascending =False)</li><li>x,LoweCase</li></ol><h3>What are collections in Scala and how can you find the maximum number in maxArray If val maxArray = Array(10,40,60,05,50)?</h3><blockquote><em>, collections are data structures that can hold and manipulate multiple values. There are several types of collections, including arrays, lists, sets, maps, and more. Collections in Scala are categorized into mutable and immutable collections. Mutable collections can be modified after creation, while immutable collections cannot be changed once created.</em></blockquote><blockquote><em>In your specific example, </em><em>maxArray is defined as an </em><em>Array, which is a mutable collection in Scala. An array is a fixed-size, indexed collection that stores elements of the same type.</em></blockquote><blockquote><em>To find the maximum number in the </em><em>maxArray, you can use various approaches, including iterating through the array and keeping track of the maximum value or using built-in functions. Here&#39;s how you can find the maximum number in your </em><em>maxArray</em></blockquote><p>curly braces with no semicolon</p><pre>val maxArray = Array(10, 40, 60, 5, 50)</pre><pre>// Using a loop to find the maximum value<br>var max = maxArray(0) // Assume the first element is the maximum</pre><pre>for (num &lt;- maxArray) {<br>  if (num &gt; max) {<br>    max = num<br>  }<br>}</pre><pre>println(s&quot;The maximum number in maxArray is: $max&quot;)</pre><p>Alterntively:</p><pre>val maxArray = Array(10, 40, 60, 5, 50)</pre><pre>val max = maxArray.max<br>println(s&quot;The maximum number in maxArray is: $max&quot;)</pre><h3>what is reduce and how it works to reduce works with Array in finding max in it?</h3><blockquote><em>In Scala, the </em><em>reduce method is used to perform a binary operation on the elements of a collection and return a single result. The </em><em>reduce function works by successively applying a binary function to the elements of the collection in a cumulative manner. It starts with the first two elements of the collection, then takes the result of that operation and combines it with the next element, and so on, until a single result is obtained.</em></blockquote><blockquote><em>To find the maximum number in an array using </em><em>reduce, you can define a binary function that compares two numbers and returns the maximum of the two. Here&#39;s how you can use </em><em>reduce to find the maximum number in an array:</em></blockquote><pre>val maxArray = Array(10, 40, 60, 5, 50)</pre><pre>// Using reduce to find the maximum number<br>val max = maxArray.reduce((a, b) =&gt; if (a &gt; b) a else b)</pre><pre>println(s&quot;The maximum number in maxArray is: $max&quot;)</pre><p>In Scala, a “companion object” and a “singleton object” are related concepts that work together to provide an interesting feature of the language. They are often associated with classes. Let me explain the difference between them with a programming example.</p><p>Companion Object: A companion object is an object that has the same name as a class and is defined in the same source file as the class. It is typically used for defining static or class-level members, methods, and behaviors related to the class. A companion object can access the private members of the associated class, and the class can access the members and methods of its companion object.</p><p>Singleton Object: A singleton object is an object that is defined without a corresponding class. It’s a single instance object that serves as a standalone entity. Singleton objects are used for defining utility functions or for grouping related functions and values. They do not have an associated class and cannot be instantiated.</p><p>Here’s an example to illustrate the difference:</p><pre>Singleton Object<br>A singleton object is an object that is defined using the object keyword and has a single instance. It is similar to a class with only one instance. Singleton objects are often used for utility methods or constants.<br><br>Here’s a brief example:<br><br>scala<br>Copy code<br>object SingletonExample {<br>  val greeting: String = &quot;Hello, Singleton!&quot;<br><br>  def greet(): Unit = {<br>    println(greeting)<br>  }<br>}<br><br>// Usage<br>SingletonExample.greet() // Prints: Hello, Singleton!<br>In this example:<br><br>SingletonExample is a singleton object.<br>It contains a greeting value and a greet method.<br>You can call the greet method directly on SingletonExample without needing to create an instance.<br><br><br><br><br>Companion Object<br>A companion object is an object that shares the same name as a class and is defined in the same file as that class. It can access the private members of the class and vice versa. Companion objects are often used to define factory methods or apply methods to create instances of the class.<br><br>Here’s a brief example:<br><br>scala<br>Copy code<br>class Person(val name: String, val age: Int) {<br>  // Instance method<br>  def greet(): Unit = {<br>    println(s&quot;Hi, my name is $name and I am $age years old.&quot;)<br>  }<br>}<br><br>object Person {<br>  // Factory method<br>  def apply(name: String, age: Int): Person = new Person(name, age)<br>}<br><br>// Usage<br>val person = Person(&quot;Alice&quot;, 30) // Calls the apply method in the companion object<br>person.greet() // Prints: Hi, my name is Alice and I am 30 years old.<br>In this example:<br><br>Person is a class with a constructor that takes name and age.<br>The companion object Person contains an apply method, which acts as a factory method to create instances of the Person class.<br>You can create a new Person instance using Person(&quot;Alice&quot;, 30) instead of new Person(&quot;Alice&quot;, 30).<br>The apply method in the companion object simplifies object creation and makes the code more concise.</pre><p>What are traits and explain with a programming example?</p><p>In Scala, a trait is a special kind of class that defines a set of abstract methods and concrete fields or methods that can be mixed into classes to provide reusable behavior. Traits are similar to Java interfaces but can also contain concrete methods and fields, making them a more powerful and flexible mechanism for code reuse and composition.</p><p>Here’s an example to explain traits in Scala:</p><pre>// Define a trait called Printable<br>trait Printable {<br>  // Abstract method<br>  def print: Unit</pre><pre>  // Concrete method<br>  def logMessage(message: String): Unit = {<br>    println(s&quot;Logging: $message&quot;)<br>  }<br>}</pre><pre>// Create a class that extends the Printable trait<br>class Document(title: String) extends Printable {<br>  def print: Unit = println(s&quot;Printing document: $title&quot;)<br>}</pre><pre>// Create another class that extends the Printable trait<br>class Email(subject: String) extends Printable {<br>  def print: Unit = println(s&quot;Printing email: $subject&quot;)<br>}</pre><pre>object Main extends App {<br>  val doc = new Document(&quot;Sample Document&quot;)<br>  doc.print<br>  doc.logMessage(&quot;Document printed&quot;)</pre><pre>  val email = new Email(&quot;Important Email&quot;)<br>  email.print<br>  email.logMessage(&quot;Email printed&quot;)<br>}</pre><p>29th jan:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*heZ9NurJrhWX8_FPVVMEmg@2x.jpeg" /></figure><p>31st jan:</p><p>altou i forot .map jo ki yaad rakna caoye ji .map ya .filter to must i must ai scla ya pyspark me</p><p>abi bi nice wala code alat ai batao</p><p>aar list bana rae o to list liko no square brackets ere…list ka l capital me oa</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/322/1*M-K_uKGfHBsYzpPqoC3NNQ.png" /></figure><p>final one:</p><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val a = List(2, 3, 4, 5, 6)<br>    val b = a.map(num =&gt; num * num)</pre><pre>    println(&quot;Squared elements:&quot;)<br>    println(b)<br>  }<br>}</pre><p>2nd one :</p><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val start = 1<br>    val end = 10<br>    val integers = (start to end).toList<br>    val b = integers.map(num =&gt; num * num)</pre><pre>    println(&quot;Squared elements:&quot;)<br>    println(b)<br>  }<br>}</pre><p>even number ke liye wau upar wale me cane kar diya but usee mere pass ab true false aa raa ai lsist nai aa rai</p><p>reson filter laana ta map laa liya</p><p>isse bi fark padta ai</p><p>incorrect one: ise teue ya false aayea bas</p><pre>object JDoodle {<br>    def main(args:Array[String]):Unit = {<br>  <br>        val a = List(2,3,4,5,6)<br>        val b = a.map(num =&gt; num*num)<br> <br>        println(b)<br>}<br>}</pre><p>correct one</p><pre>// Create a list of even numbers from a given list of integers</pre><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val start = 1<br>    val end = 10<br>    val integers = (start to end).toList<br>    val b = integers.filter(num =&gt; num %2 == 0)</pre><pre>    println(&quot;even elements:&quot;)<br>    println(b)<br>  }<br>}</pre><pre>/Create a list of all the vowels in a given sentence</pre><p>incorrect</p><pre>class jdoodle {<br>   def main(ars:stin[array]): unit {<br>       <br>       val strin = &quot;ankitjunnu is a star&quot;<br>       val ceck  = strin.filter(a =&gt; a  in aeiou).toList<br>   }<br>}</pre><p>correct</p><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World!&quot;<br>    val vowels = inputString.filter(c =&gt; &quot;aeiouAEIOU&quot;.contains(c))</pre><pre>    println(&quot;Vowels in the string:&quot;)<br>    println(vowels.toList)<br>  }<br>}</pre><p>anoter correct metod, wic is more convineient</p><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World!&quot;<br>    val vowels = inputString.filter(c =&gt; c &lt;- &quot;aeiouAEIOU&quot;)<br><br>    println(&quot;Vowels in the string:&quot;)<br>    println(vowels.toList)<br>  }<br>}</pre><p>incorrect:</p><p>sabse bada fark yai ai ai ki jab umne sentence ko split kiya tab umne map ya filter use nai kiya ab samaj me aay kuon nai kiya</p><p>kuonki umere ek ek caracter ko lelke kya katrna ai jab ume strin is split karni ai wats te need of caracter ten</p><pre>//Create a list of all the words in a given sentence</pre><pre>object jDoodle{<br>    def main(args:String[Array]):Unit = {<br>        val strin = &quot;just manifest&quot;<br>        val asd= strin.split(&quot;//s+&quot;)<br>        <br>        println(asd)<br>        <br>    }<br>}</pre><p>isme array ke andar strin aayea naaki ulta</p><p>also asd ko list me convert to karo</p><p>3rd reular expresion used in split is incooreect</p><pre>//Create a list of all the words in a given sentence</pre><pre>object JDoodle {<br>      def main(args: Array[String]): Unit = {<br>        val strin = &quot;just manifest&quot;<br>        val asd= strin.split(&quot;\\s+&quot;)<br>        <br>        println(asd.toList)<br>        <br>    }<br>}</pre><h3>Create a list of all the positive numbers from a given list of integers.</h3><p>correct:</p><pre>//. </pre><pre>object JDoodle {<br>      def main(args: Array[String]): Unit = {<br>          val start = -1<br>          val end =  10<br>          val asd = (start to end).toList<br>          <br>          val list1 = asd.filter(num =&gt; num&gt;0)<br>        <br>        println(list1.toList)<br>        <br>    }<br>}</pre><h3>Create a list of all the uppercase characters from a given string.</h3><p>c.isUpper ke baad bracket nai aayei</p><pre>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World!&quot;<br>    val vowels = inputString.filter(c =&gt; c.isUpper)</pre><pre>    println(&quot;upperletter:&quot;)<br>    println(vowels.toList)<br>  }<br>}</pre><h3>Create a list of all the first letters of words in a given sentence.</h3><p>isko pyton me kaise karta</p><p>usme input baad me aati ai list compreension me</p><pre>sentence = &quot;This is a sample sentence.&quot;<br>first_letters = [word[0] for word in sentence.split()]</pre><p>isme kaise kare</p><p>one moment to stop abi tak um jitne bi input de rae ai wo um kisi word ko nai balki eac caracter ko de rae ai and caracter ko app first word als nai kar sakte</p><p>The issue in your code is with the expression c(0). In Scala, a Char instance cannot be accessed with parentheses like an array or a string to get its individual character. Instead, you should use the charAt method of the String class to access characters by index. Additionally, to extract the first letter of each word, you need to split the input string into words first.</p><p>aur isliye pele split karna padea</p><p>split karo witout usin map ten uske baad map laao</p><p>also one more tin</p><blockquote>the underscore (_) is a shorthand notation often used in combination with higher-order functions like map, filter, foreach, etc., to represent an anonymous function parameter.</blockquote><blockquote>In the expression map(_.charAt(0)), the underscore _ is used as a placeholder for each element of the collection during the execution of the map function. It is equivalent to writing a named parameter explicitly.</blockquote><blockquote>Here’s a breakdown of what’s happening in map(_.charAt(0)):</blockquote><blockquote>_: This underscore represents each element of the collection as the function is applied.</blockquote><blockquote>charAt(0): This is a method call on each element of the collection. It extracts the first character of the element.</blockquote><blockquote>So, map(_.charAt(0)) can be read as &quot;for each element, extract its first character&quot;. It&#39;s a concise way to write an anonymous function without explicitly naming the parameter.</blockquote><p>usin you should use the charAt method of the String class to access characters by index</p><pre>//Create a list of all the first letters of words in a given sentence.<br>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World  its junnu!&quot;<br>    val vowels = inputString.map(c =&gt; c.split(&quot;//s+&quot;)).map(_.charAt(0))<br><br>    println(&quot;upperletter:&quot;)<br>    println(vowels.toList)<br>  }<br>}</pre><p>aain incorrect, you sould use one tin very secifically tat is split is not used wit map.</p><pre>//Create a list of all the first letters of words in a given sentence.<br>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World  its junnu!&quot;<br>    val firstLetters = inputString.split(&quot;\\s+&quot;).map(_.charAt(0))<br><br>    println(&quot;First letters of words:&quot;)<br>    println(firstLetters.toList)<br>  }<br>}<br><br></pre><h3>or</h3><pre>val firstLetterList = sentence.split(&quot;\\s+&quot;).map(word =&gt; word.head).toList</pre><h3>//Create a list of all the unique words from a given sentence</h3><p>in pyton\\</p><blockquote>unique_words = list(set(sentence.split()))</blockquote><p>yw scala ai , wy to foret .set nai .toSet aayea aur wo bi split karne ke baad</p><pre>//Create a list of all the unique words from a given sentence<br>object JDoodle {<br>  def main(args: Array[String]): Unit = {<br>    val inputString = &quot;Hello, World  its junnu junnu !&quot;<br>    val unique = inputString.split(&quot;\\s+&quot;).toSet<br><br>    println(&quot;unique letters of words:&quot;)<br>    println(unique.toList)<br>  }<br>}</pre><p><strong>Create a list of all the permutations of a given string using list comprehension.</strong></p><p>permutation ne string me dall diya (‘a’,’b’,’c’)</p><pre>input_string = &quot;abc&quot;<br>permutations_list = [&#39;&#39;.join(p) for p in permutations(input_string)]</pre><p>scla me permutation metod nai ai to ume bilkul step by step karna ofa</p><p>for loop in pyton kaisa ot aia</p><pre>class Solution:<br> def twoSum(self, nums: List[int], target: int) -&gt; List[int]:<br> a=[]<br> for i in range(len(nums)):<br> for j in range(i+1,len(nums)):<br> if (nums[i]+nums[j]==target):<br> a.append(i)<br> a.append(j)<br> break<br> else:<br> continue <br> return a<br><br></pre><p><strong>scala me list ki jaa indices aayea</strong></p><p>jaise u pyton me append use karte te yaaa pe yield use oa</p><pre>def permutations(s: String): List[String] = {<br>  if (s.isEmpty) List(&quot;&quot;)<br>  else {<br>    for {<br>      i &lt;- s.indices.toList<br>      c = s(i)<br>      rest = s.substring(0, i) + s.substring(i + 1)<br>      perm &lt;- permutations(rest)<br>    } yield c + perm<br>  }<br>}<br><br>// Example usage:<br>val inputString = &quot;abc&quot;<br>val result = permutations(inputString)<br>println(result)</pre><pre>Outer Loop Iteration 1: i = 0, c = &#39;a&#39;, rest = &quot;bc&quot;<br><br>Inner Loop: Generates permutations of &quot;bc&quot; (the string with &#39;a&#39; removed).<br>For each permutation perm, concatenates &#39;a&#39; with perm and yields the result.<br>Example: &quot;a&quot; + &quot;bc&quot;, &quot;a&quot; + &quot;cb&quot;.<br>Outer Loop Iteration 2: i = 1, c = &#39;b&#39;, rest = &quot;ac&quot;<br><br>Inner Loop: Generates permutations of &quot;ac&quot; (the string with &#39;b&#39; removed).<br>For each permutation perm, concatenates &#39;b&#39; with perm and yields the result.<br>Example: &quot;b&quot; + &quot;ac&quot;, &quot;b&quot; + &quot;ca&quot;.<br>Outer Loop Iteration 3: i = 2, c = &#39;c&#39;, rest = &quot;ab&quot;<br><br>Inner Loop: Generates permutations of &quot;ab&quot; (the string with &#39;c&#39; removed).<br>For each permutation perm, concatenates &#39;c&#39; with perm and yields the result.<br>Example: &quot;c&quot; + &quot;ab&quot;, &quot;c&quot; + &quot;ba&quot;.<br>The resulting list of permutations would be List(&quot;abc&quot;, &quot;acb&quot;, &quot;bac&quot;, &quot;bca&quot;, &quot;cab&quot;, &quot;cba&quot;).<br><br><br><br></pre><pre>Recursive Call for &quot;bc&quot;:<br><br>Outer Loop Iteration 1: i = 0, c = &#39;b&#39;, rest = &quot;c&quot;<br>Inner Loop (Recursion): Generates permutations of &quot;c&quot;.<br>The function is recursively called with &quot;c&quot; as the input.<br>Since &quot;c&quot; has only one character, the base case is reached, and the function returns List(&quot;c&quot;).<br>Concatenation with &#39;b&#39;: Concatenates &#39;b&#39; with each permutation of &quot;c&quot;.<br>Resulting permutations: &quot;bc&quot;, &quot;cb&quot;<br>Concatenation with &#39;a&#39;: Concatenates &#39;a&#39; with each permutation of &quot;bc&quot;.<br><br>Resulting permutations: &quot;abc&quot;, &quot;acb&quot;<br>So, in the recursive call for &quot;bc&quot;, it generates permutations of the substring &quot;c&quot; (which has only one character), and then it combines each of these permutations with &#39;b&#39; to form new permutations. Finally, these permutations are combined with &#39;a&#39; to form the final permutations of the original string &quot;abc&quot;.</pre><p>same all te above codes in for-comprehensions</p><p>like in previous we use map or filter and ten iven input =&gt; output….in te for compreension is in for { isme yaa pe list kon si aai kaa se input leni ai, koi if condition ai } output me yield me daal dene main function</p><pre><br>Certainly! Here&#39;s how you can accomplish each task using for-comprehensions in Scala:<br><br>Create a list of squared integers from a given list of integers:<br>scala<br>Copy code<br>val squaredList = for {<br>  num &lt;- originalList<br>} yield num * num<br><br>Create a list of even numbers from a given list of integers:<br>scala<br>Copy code<br>val evenList = for {<br>  num &lt;- originalList<br>  if num % 2 == 0<br>} yield num<br>Create a list of vowels from a given string:<br>scala<br>Copy code<br>val vowelList = for {<br>  char &lt;- inputString<br>  if &quot;aeiouAEIOU&quot;.contains(char)<br>} yield char<br>Create a list of all the words in a given sentence:<br>scala<br>Copy code<br>val wordList = for {<br>  word &lt;- sentence.split(&quot;\\s+&quot;)<br>} yield word<br>Create a list of all the positive numbers from a given list of integers:<br>scala<br>Copy code<br>val positiveList = for {<br>  num &lt;- originalList<br>  if num &gt; 0<br>} yield num<br>Create a list of all the uppercase characters from a given string:<br>scala<br>Copy code<br>val uppercaseList = for {<br>  char &lt;- inputString<br>  if char.isUpper<br>} yield char<br>Create a list of all the first letters of words in a given sentence:<br>scala<br>Copy code<br>val firstLetterList = for {<br>  word &lt;- sentence.split(&quot;\\s+&quot;)<br>} yield word.head<br>Create a list of all the unique words from a given sentence:<br>scala<br>Copy code<br>val uniqueWordList = for {<br>  word &lt;- sentence.split(&quot;\\s+&quot;).toSet<br>} yield word<br>Create a list of all the anagrams of a given word from a list of words:<br>scala<br>Copy code<br>val anagramList = for {<br>  word &lt;- wordList<br>  if word.permutations.toList.contains(givenWord)<br>} yield word<br>Create a list of all the permutations of a given string using list comprehension:<br>scala<br>Copy code<br>val permutationList = for {<br>  perm &lt;- givenString.permutations<br>} yield perm</pre><p><strong>11. Create a list of all the words that start and end with the same letter from a given list of words using list comprehension.</strong></p><pre>word_list = [&quot;radar&quot;, &quot;hello&quot;, &quot;level&quot;, &quot;world&quot;, &quot;noon&quot;]<br>same_letter_words = [word for word in word_list if word[0] == word[-1]]</pre><p><strong>12. Create a list of all the numbers that are divisible by 3 and 5 from a given list of integers using list comprehension.</strong></p><pre>original_list = [15, 9, 25, 7, 30, 18]<br>divisible_by_3_and_5 = [x for x in original_list if x % 3 == 0 and x % 5 == 0]</pre><p><strong>13. Convert a given string to lowercase</strong></p><pre>input_string = &quot;Hello, World!&quot;<br>lowercase_string = input_string.lower()</pre><p><strong>14. Reverse a given string</strong></p><p>without split…it is same take character ..in a string it will always go for characters and not for words.</p><pre>input_string = &quot;Hello, World!&quot;<br>reversed_string = input_string[::-1]<br></pre><pre>output:!dlroW ,olleH</pre><p>with split, it creates a list</p><pre>input_string = &quot;Hello, World!&quot;<br>input_string2 = input_string.split()<br>reversed_string = input_string2[::-1]<br>reversed_string</pre><p><strong>15. Remove all the vowels from a given string</strong></p><p>first using regex:</p><p>re.sub() is a function from the re module in Python, which stands for &quot;regular expression substitution.&quot; It is used to perform substitution or replacement of text patterns in a string using regular expressions.</p><pre>import re</pre><pre>input_string = &quot;Hello, World!&quot;<br>no_vowels_string = re.sub(r&#39;[aeiouAEIOU]&#39;, &#39;&#39;, input_string)</pre><p><strong>16. Find all the digits in a given string</strong></p><p>if you are usin ! iska matlan nai lana isko</p><pre>import re<br>text = &quot;Hello, 123 World! 456&quot;<br>digits = re.findall(r&#39;\d&#39;, text)<br>print(digits)</pre><p><strong>17. Remove all the whitespace characters from a given string</strong></p><pre>text = &quot;This is a sample text with whitespace&quot;<br>text = &#39;&#39;.join(text.split())<br>print(text)</pre><figure><img alt="" src="https://cdn-images-1.medium.com/max/403/0*ZQgQrdSksmJDpczY.png" /></figure><p><strong>18. Create a string of all the characters in a given list of strings</strong></p><pre>list_of_strings = [&quot;Hello&quot;, &quot;World&quot;]<br>combined_string = &#39;&#39;.join(list_of_strings)<br>print(combined_string)</pre><p><strong>19. Remove all the punctuation characters from a given string</strong></p><pre>import string<br>text = &quot;Hello, World! This is a sample text.&quot;<br>text = &#39;&#39;.join(char for char in text if char not in string.punctuation)<br>print(text)</pre><p><strong>20. Count the number of times each character appears in a given string</strong></p><p>isme char_set humne empty se start kiya hai</p><pre>text = &quot;This is a sample text.&quot;<br>char_set = set()<br>char_count = {}</pre><pre>for char in text:<br>    if char.isalpha():<br>        if char in char_set:<br>            char_count[char] += 1<br>        else:<br>            char_set.add(char)<br>            char_count[char] = 1</pre><pre># Display character counts<br>for char, count in char_count.items():<br>    print(f&quot;&#39;{char}&#39;: {count}&quot;)</pre><blockquote><em>iss wale case me hume char string pehle se di hui hai humne wo dhundhni hai</em></blockquote><blockquote><strong><em>isme humne for loop kle andar jo bracket hoti hai usse wo pata chala ki kitni iteration rakhni hai uske andar</em></strong></blockquote><h3>len(mystring) - len(character_to_check) + 1 calculates the number of iterations required in the for loop. Here&#39;s why this specific value is used:</h3><pre>def check_a(mystring):<br>    character_to_check = &quot;Hell&quot;<br>    found = False</pre><pre>    for i in range(len(mystring) - len(character_to_check) + 1):<br>        if mystring[i:i + len(character_to_check)] == character_to_check:<br>            found = True<br>            break</pre><pre>    if found:<br>        print(f&quot;The substring &#39;{character_to_check}&#39; is present in the string.&quot;)<br>    else:<br>        print(f&quot;The substring &#39;{character_to_check}&#39; is NOT present in the string.&quot;)</pre><pre>check_a(&quot;Hello, world!&quot;)</pre><p>other metod to check instance</p><pre>def check_a(mystring):<br>    character_to_check = &quot;Hell&quot;</pre><pre>    index = mystring.find(character_to_check)</pre><pre>    if index != -1:<br>        print(f&quot;The substring &#39;{character_to_check}&#39; is present in the string at index {index}.&quot;)<br>    else:<br>        print(f&quot;The substring &#39;{character_to_check}&#39; is NOT present in the string.&quot;)</pre><pre>check_a(&quot;Hello, world!&quot;)</pre><p><strong>21. Find the most common character in a given string</strong></p><p><strong>22. Capitalize the first letter of each word in a given string</strong></p><p><strong>23. Find all the anagrams of a given word from a list of words</strong></p><p><strong>24. Create a string of all the unique characters in a given string</strong></p><p>strin.distinct will return strin only</p><pre>object JDoodle {<br>  val string = &quot;i love myself&quot;<br> <br>  def main(args: Array[String]) {<br>    val uniqueChars = string.distinct  or  println(uniqueChars.toList)<br>    println(uniqueChars)<br>  }<br>}</pre><p><strong>25. Find the longest palindrome in a given string</strong></p><p><strong>26. Find all the substrings of a given string</strong></p><p><strong>27. Create a string of all the words in a given sentence in reverse order</strong></p><p><strong>28. Create an array of integers from a given list of integers.</strong></p><p><strong>29. Find the sum of all the elements in a given array of integers.</strong></p><p><strong>30. Find the maximum element in a given array of integers.</strong></p><p><strong>31. Find the minimum element in a given array of integers.</strong></p><p><strong>32. Create an array of all the even numbers from a given array of integers.</strong></p><p>abi neece walo nme</p><h3>max, larest, sorted, reverse, take(2), last</h3><p>ye naye keywords aane wale ai</p><p><strong>33. Find the second largest element in a given array of integers.</strong></p><p><strong>34. Find the second smallest element in a given array of integers.</strong></p><p><strong>35. Create a new array from two given arrays by alternating their elements.</strong></p><p><strong>36. Create a new array from a given array by removing all the duplicates.</strong></p><p><strong>37. Create a new array from a given array by reversing its elements.</strong></p><p><strong>38. Create a new array from a given array by shuffling its elements randomly.</strong></p><p><strong>39. Create a new array from a given array by selecting only the elements that are divisible by 3 and 5.</strong></p><p><strong>40. Find the intersection of two given lists.</strong></p><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=9fb6eeb8edc4" width="1" height="1" alt="">]]></content:encoded>
        </item>
        <item>
            <title><![CDATA[udemy courses]]></title>
            <link>https://medium.com/@ankitjuneja/udemy-courses-aa7dcfb548ff?source=rss-31751420d0e2------2</link>
            <guid isPermaLink="false">https://medium.com/p/aa7dcfb548ff</guid>
            <dc:creator><![CDATA[Ankit Juneja]]></dc:creator>
            <pubDate>Sat, 13 Jan 2024 22:35:02 GMT</pubDate>
            <atom:updated>2024-01-16T11:06:32.695Z</atom:updated>
            <content:encoded><![CDATA[<p>aws_data_engineer_DEA_C01</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1021/1*jj04RFLesIGWjEkNI4UFLA.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*WWM1_7u0-LhD4duUpX6TCg.png" /></figure><p>DATA-MESH</p><p>MORE ABOUT GOVERNANAVCE</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/912/1*oPbD415SfrNyVTYVYDOQxg.png" /></figure><p>2 IN ETL ORCESTRATION SERVICESS LIKE</p><p>EVENTBIDE AND STEP FUNCTIONS I NEED TO WORK ABOUT</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/552/1*WqsO7ukJzTqnF3ksS4joew.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/674/1*TIlC4tGY3wASbbG1QG7XPQ.png" /></figure><p>CSV. JSON,</p><p>AVRO (SERIALISATION)</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/676/1*kfLZLS4sOqGyqro8t82hqQ.png" /></figure><p>PARQUET(ANALYTICS)</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/687/1*6Hhqm0WV8Ow0r1w6Yc2tiw.png" /></figure><p>DATA MODELLIN</p><p>IT AS ONE TERM TO LINEAA TE DATA , TE TERMINOLIY USED IS IMPORTANT</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*ofc1Dn5E7z_q3wKp1x5FJw.png" /></figure><p>AMAZON NEPTUNE IS A RAPICAL DATABASE</p><p>SAMPLIN</p><p>RANDOM WERE STRATIFIES VS SYSTEM</p><p>STRATIFIES . 2 FROM EAC SUBROUP</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/653/1*YUIs3yD3G9bKsGZYSglCpg.png" /></figure><p>SYSTEMATIC SAMPLIN….AFTER EVERY 3</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/655/1*dCK3hOlmKk2n5TrOamDpvg.png" /></figure><p>DATA SKEWNESS = UNEQUAL DISTRIBUTION</p><p>IMDB BRADD PITT EXAMPLE</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/713/1*PiOTrjnVipf6WW5UDTW_wg.png" /></figure><p>SOLUTION:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/720/1*WdZPqsiWEgQ6W0l9YacNCA.png" /></figure><p>VALIDATION OF DATA</p><p>CONSISTENCYMEANS OME MOVIES RATE OUT OF 5 SOME OUT OF 10 AND WEN TEY CLUUBEDD TOETER</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/709/1*-k6fZW4Qf_5zx5imIzlMvQ.png" /></figure><p>SQL PIVOTIN AVE NOT USED NAY:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*EeeyXWW1uQgsedb1lN3Utw.png" /></figure><p>OR</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1007/1*ekTYc-rl2G-ICgUBxpaK8w.png" /></figure><p>IT</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/1024/1*PYJBecRf25ZxYEcHfhh6-g.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/550/1*RfIx54SV8wHXGzRmjYH4fg.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/831/1*IvEwb_FfJ54lZ1bxF2H_wA.png" /></figure><p>STORAE</p><p>FINALLY UNDERSTANDIN SOME BASICS ABOUT S3</p><p>numver alpabets and ypens are allowed</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/942/1*zHqggod6b6ZmSNZPAREiJA.png" /></figure><p>2ND</p><p>acl permission remember</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/978/1*2mMvtralQ1szgamHcJlTpQ.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/641/1*5Nf-FIh1JJBDsQMNETVgtA.png" /></figure><p>s3 buckets me imae object s3 me khola to chal raha hai but outside it not workin</p><p>security issues</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/973/1*ootbqqybiOYOPryLcKi21Q.png" /></figure><p>in tis example we ave iven public read in json object</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/972/1*3pkmyp_n9ZJeYPsN3wXOaA.png" /></figure><p>few examples</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/331/1*rIJ5MyPNKwF1otwkKT7b2A.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/636/1*Rqqmc96seISB021-90cuyA.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/677/1*i8DTZTlpQjgLz29zKrDOpw.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/635/1*Fp3cGEaIg4Due2VlUaairQ.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/661/1*DD_79xZe4duW2ltrH3EHxg.png" /></figure><p>tree types</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/626/1*AL6rDojKwei-ppv_pSnucg.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/880/1*sutYuybEA5YRHA2L1ddJ7w.png" /></figure><p>databases:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/817/1*t-AuhFue8oTRX3VtHvrJoQ.png" /></figure><p>encryption in s3:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/884/1*E9QaVga6omB5TnNtBPOa3Q.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/491/1*DdjMcrm1pmnvkYB1JIqfUA.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/478/1*NECotWSRlMmjhXu-oQc5kQ.png" /></figure><p>access points</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/522/1*wj4FIf8Da9R8hWJBkk6kqw.png" /></figure><p>ebs volume:</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/531/1*zipEzUyxDopWTWTaZ4t08Q.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/477/1*00vw43qvKRPHha24RxPxgQ.png" /></figure><p>backup</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/651/1*U-PXH-1ENYovi9Z8va1rjw.png" /></figure><h3>databases</h3><figure><img alt="" src="https://cdn-images-1.medium.com/max/823/1*-adde9YKmVqoVN15GrJJ5g.png" /></figure><p>answer is b …lobal</p><figure><img alt="" src="https://cdn-images-1.medium.com/max/831/1*yHHv0qoVXuLbfrHWT4Gtgg.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/814/1*slYEdNqgbxFArYkr49orwg.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/820/1*PyhzSZQc-qGh9jwnGdZt9w.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/814/1*gBpITLT_vtqAktADM7-45w.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/855/1*AxAQNP9Xi1-tJjlt1H-d1Q.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/797/1*7pCGpOlwKUNH9fss78ReoA.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/839/1*igttuwL2GVSu4wHpy5-IBQ.png" /></figure><figure><img alt="" src="https://cdn-images-1.medium.com/max/855/1*UsJrd9jeVl1PoFd03O6G5A.png" /></figure><img src="https://medium.com/_/stat?event=post.clientViewed&referrerSource=full_rss&postId=aa7dcfb548ff" width="1" height="1" alt="">]]></content:encoded>
        </item>
    </channel>
</rss>