MDEE-342: limit feed full reindex by days.

Author: obarcelonap

DataExporter/Model/Indexer/AllIdsResolver.php

@@ -7,6 +7,7 @@
 
 namespace Magento\DataExporter\Model\Indexer;
 
+use DateTime;
 use Magento\Framework\App\ResourceConnection;
 use Magento\Framework\DB\Select;
 
@@ -45,8 +46,10 @@ public function getAllIds(FeedIndexMetadata $metadata): ?\Generator
         $connection = $this->resourceConnection->getConnection();
         $lastKnownId = -1;
         $continueReindex = true;
+        $dateTime = date_create();
+
         while ($continueReindex) {
-            $ids = $connection->fetchAll($this->getIdsSelect((int)$lastKnownId, $metadata));
+            $ids = $connection->fetchAll($this->getIdsSelect((int)$lastKnownId, $metadata, $dateTime));
             if (empty($ids)) {
                 $continueReindex = false;
             } else {
@@ -61,25 +64,37 @@ public function getAllIds(FeedIndexMetadata $metadata): ?\Generator
      *
      * @param int $lastKnownId
      * @param FeedIndexMetadata $metadata
+     * @param DateTime $dateTime
      * @return Select
      */
-    private function getIdsSelect(int $lastKnownId, FeedIndexMetadata $metadata): Select
+    private function getIdsSelect(int $lastKnownId, FeedIndexMetadata $metadata, DateTime $dateTime): Select
     {
         $connection = $this->resourceConnection->getConnection();
         $tableName = $this->resourceConnection->getTableName($metadata->getSourceTableName());
 
         $columnExpression = sprintf('s.%s', $metadata->getSourceTableIdentityField());
         $whereClause = sprintf('s.%s > ?', $metadata->getSourceTableIdentityField());
 
-        return $connection->select()
+        $select = $connection->select()
             ->from(
                 ['s' => $tableName],
                 [
                     $metadata->getFeedIdentity() => 's.' . $metadata->getSourceTableField(),
                     self::IDENTITY_FIELD_NAME => 's.' . $metadata->getSourceTableIdentityField()
                 ]
             )
-            ->where($whereClause, $lastKnownId)
+            ->where($whereClause, $lastKnownId);
+
+        if ($metadata->getFullReIndexDaysLimit() != 0) {
+            $select->where(sprintf(
+                "s.%s >= DATE_SUB(STR_TO_DATE('%s', '%%Y-%%m-%%d %%H:%%i:%%s'), INTERVAL %d DAY)",
+                $metadata->getSourceTableFieldOnFullReIndexDaysLimit(),
+                $dateTime->format('Y-m-d H:i:s'),
+                $metadata->getFullReIndexDaysLimit()
+            ));
+        }
+
+        return $select
             ->order($columnExpression)
             ->limit($metadata->getBatchSize());
     }

DataExporter/Model/Indexer/FeedIndexMetadata.php

@@ -59,6 +59,16 @@ class FeedIndexMetadata
      */
     private $sourceTableIdentityField;
 
+    /**
+     * @var int
+     */
+    private $fullReindexDaysLimit;
+
+    /**
+     * @var string
+     */
+    private $sourceTableFieldOnFullReIndexDaysLimit;
+
     /**
      * @param string $feedName
      * @param string $sourceTableName
@@ -79,7 +89,9 @@ public function __construct(
         string $feedTableField,
         array $feedTableMutableColumns,
         int $batchSize = 100,
-        string $sourceTableIdentityField = null
+        string $sourceTableIdentityField = null,
+        int $fullReIndexDaysLimit = 0,
+        string $sourceTableFieldOnFullReIndexDaysLimit = 'updated_at'
     ) {
         $this->feedName = $feedName;
         $this->sourceTableName = $sourceTableName;
@@ -90,6 +102,8 @@ public function __construct(
         $this->feedTableMutableColumns = $feedTableMutableColumns;
         $this->batchSize = $batchSize;
         $this->sourceTableIdentityField = $sourceTableIdentityField ?? $sourceTableField;
+        $this->fullReindexDaysLimit = $fullReIndexDaysLimit;
+        $this->sourceTableFieldOnFullReIndexDaysLimit = $sourceTableFieldOnFullReIndexDaysLimit;
     }
 
     /**
@@ -181,4 +195,22 @@ public function getFeedTableMutableColumns(): array
     {
         return $this->feedTableMutableColumns;
     }
+
+    /**
+     * Determines the amount of days back when triggering a full reindex
+     * @return int the amount in days, 0 means no limit
+     */
+    public function getFullReIndexDaysLimit(): int
+    {
+        return $this->fullReindexDaysLimit;
+    }
+
+    /**
+     * Table field name to use when full reindex is limited by days back (see fullReindexDaysLimit)
+     * @return string the field name
+     */
+    public function getSourceTableFieldOnFullReIndexDaysLimit(): string
+    {
+        return $this->sourceTableFieldOnFullReIndexDaysLimit;
+    }
 }

SalesOrdersDataExporter/etc/di.xml

@@ -19,6 +19,8 @@
             <argument name="feedTableMutableColumns" xsi:type="array">
                 <item name="feed_data" xsi:type="string">feed_data</item>
             </argument>
+            <argument name="fullReIndexDaysLimit" xsi:type="number">7</argument>
+            <argument name="sourceTableFieldOnFullReIndexDaysLimit" xsi:type="string">updated_at</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\SalesOrdersDataExporter\Model\Indexer\OrdersFeedIndexer" type="Magento\DataExporter\Model\Indexer\FeedIndexer">