From c82e6f90412c59050b6f51aa89d9420d75333eef Mon Sep 17 00:00:00 2001 From: Leonid Poluianov Date: Tue, 18 Feb 2025 16:25:51 -0600 Subject: [PATCH 1/4] MDEE-974: Product Feed index taking long time to execute --- .../Model/Provider/Products.php | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/CatalogDataExporter/Model/Provider/Products.php b/CatalogDataExporter/Model/Provider/Products.php index 86d29226..e694c343 100644 --- a/CatalogDataExporter/Model/Provider/Products.php +++ b/CatalogDataExporter/Model/Provider/Products.php @@ -116,24 +116,37 @@ public function execute( } $connection = $this->resourceConnection->getConnection(); - $itemN = 0; + $storeViewItemN = []; foreach ($queryArguments as $scopeId => $productData) { $cursor = $connection->query( $this->productMainQuery->getQuery(\array_keys($productData), $scopeId ?: null) ); while ($row = $cursor->fetch()) { - $itemN++; - $mappedProducts[$row['storeViewCode']][$row['productId']] = $row; - $attributesData[$row['storeViewCode']][$row['productId']] = $productData[$row['productId']]; - if ($itemN % $metadata->getBatchSize() == 0) { - $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback, $metadata); - $mappedProducts = []; - $attributesData = []; + $storeViewCode = $row['storeViewCode']; + $productId = $row['productId']; + + if (!isset($storeViewItemN[$storeViewCode])) { + $storeViewItemN[$storeViewCode] = 0; + } + $storeViewItemN[$storeViewCode]++; + + $mappedProducts[$storeViewCode][$productId] = $row; + $attributesData[$storeViewCode][$productId] = $productData[$productId]; + + if ($storeViewItemN[$storeViewCode] % $metadata->getBatchSize() == 0 + || count($mappedProducts) % $metadata->getBatchSize() == 0) { + $this->processProducts( + ['storeViewCode' => $mappedProducts[$storeViewCode]], + ['storeViewCode' => $attributesData[$storeViewCode]], + $dataProcessorCallback, + $metadata + ); + unset($mappedProducts[$storeViewCode], $attributesData[$storeViewCode]); } } } - if ($itemN === 0) { + if (empty($storeViewItemN)) { $productsIds = \implode(',', \array_unique(\array_column($arguments, 'productId'))); $scopes = \implode(',', \array_unique(\array_column($arguments, 'scopeId'))); $this->logger->info( From e914a97f12308feb779a258cf3a37892a911db5b Mon Sep 17 00:00:00 2001 From: Leonid Poluianov Date: Wed, 19 Feb 2025 10:56:27 -0600 Subject: [PATCH 2/4] MDEE-974: Product Feed index taking long time to execute --- CatalogDataExporter/Model/Provider/Products.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CatalogDataExporter/Model/Provider/Products.php b/CatalogDataExporter/Model/Provider/Products.php index e694c343..dc3cfa4f 100644 --- a/CatalogDataExporter/Model/Provider/Products.php +++ b/CatalogDataExporter/Model/Provider/Products.php @@ -116,8 +116,8 @@ public function execute( } $connection = $this->resourceConnection->getConnection(); - $storeViewItemN = []; foreach ($queryArguments as $scopeId => $productData) { + $storeViewItemN = []; $cursor = $connection->query( $this->productMainQuery->getQuery(\array_keys($productData), $scopeId ?: null) ); From 3b95250c2d133aea29547d25628fa677f69f2d27 Mon Sep 17 00:00:00 2001 From: Leonid Poluianov Date: Wed, 19 Feb 2025 11:44:54 -0600 Subject: [PATCH 3/4] MDEE-974: Product Feed index taking long time to execute --- .../Model/Provider/Products.php | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/CatalogDataExporter/Model/Provider/Products.php b/CatalogDataExporter/Model/Provider/Products.php index dc3cfa4f..aece511e 100644 --- a/CatalogDataExporter/Model/Provider/Products.php +++ b/CatalogDataExporter/Model/Provider/Products.php @@ -137,10 +137,10 @@ public function execute( if ($storeViewItemN[$storeViewCode] % $metadata->getBatchSize() == 0 || count($mappedProducts) % $metadata->getBatchSize() == 0) { $this->processProducts( - ['storeViewCode' => $mappedProducts[$storeViewCode]], - ['storeViewCode' => $attributesData[$storeViewCode]], + $mappedProducts, + $attributesData, $dataProcessorCallback, - $metadata + $storeViewCode ); unset($mappedProducts[$storeViewCode], $attributesData[$storeViewCode]); } @@ -158,7 +158,7 @@ public function execute( ) ); } else { - $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback, $metadata); + $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback); } } @@ -179,7 +179,7 @@ public function get(array $values) : array * @param array $mappedProducts * @param array $attributesData * @param callable $dataProcessorCallback - * @param FeedIndexMetadata $metadata + * @param string|null $storeViewCode * @return void * @throws UnableRetrieveData */ @@ -187,17 +187,20 @@ private function processProducts( array $mappedProducts, array $attributesData, callable $dataProcessorCallback, - FeedIndexMetadata $metadata + string $storeViewCode = null ): void { $output = []; - - foreach ($mappedProducts as $storeCode => $products) { - $output[] = \array_map(function ($row) { - return $this->formatter->format($row); - }, \array_replace_recursive( - $products, - $this->entityEavAttributesResolver->resolve($attributesData[$storeCode], $storeCode) - )); + if (null === $storeViewCode) { + foreach ($mappedProducts as $mappedStoreViewCode => $products) { + $this->formatOutput($products, $attributesData[$storeViewCode], $output, $mappedStoreViewCode); + } + } else { + $this->formatOutput( + $mappedProducts[$storeViewCode], + $attributesData[$storeViewCode], + $output, + $storeViewCode + ); } $errorEntityIds = []; @@ -218,4 +221,28 @@ private function processProducts( $dataProcessorCallback($this->get(\array_merge(...$output))); } + + /** + * Format output + * + * @param array $products + * @param array $attributesData + * @param array $output + * @param string $storeViewCode + * @return void + * @throws UnableRetrieveData + */ + private function formatOutput( + array $products, + array $attributesData, + array &$output, + string $storeViewCode + ): void { + $output[] = \array_map(function ($row) { + return $this->formatter->format($row); + }, \array_replace_recursive( + $products, + $this->entityEavAttributesResolver->resolve($attributesData, $storeViewCode) + )); + } } From 2e092ff2fd5fdf86ed95e1acb9682006f3f371a0 Mon Sep 17 00:00:00 2001 From: Leonid Poluianov Date: Wed, 19 Feb 2025 14:28:56 -0600 Subject: [PATCH 4/4] MDEE-974: Product Feed index taking long time to execute --- CatalogDataExporter/Model/Provider/Products.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CatalogDataExporter/Model/Provider/Products.php b/CatalogDataExporter/Model/Provider/Products.php index aece511e..cd98e162 100644 --- a/CatalogDataExporter/Model/Provider/Products.php +++ b/CatalogDataExporter/Model/Provider/Products.php @@ -192,7 +192,7 @@ private function processProducts( $output = []; if (null === $storeViewCode) { foreach ($mappedProducts as $mappedStoreViewCode => $products) { - $this->formatOutput($products, $attributesData[$storeViewCode], $output, $mappedStoreViewCode); + $this->formatOutput($products, $attributesData[$mappedStoreViewCode], $output, $mappedStoreViewCode); } } else { $this->formatOutput(