Merge pull request #209 from magento-commerce/MDEE-342-limit-orders-export-full-reindex

MDEE-342: limit feed full reindex by time

Author: obarcelonap

DataExporter/Model/Indexer/AllIdsResolver.php

@@ -7,6 +7,7 @@
 
 namespace Magento\DataExporter\Model\Indexer;
 
+use DateTime;
 use Magento\Framework\App\ResourceConnection;
 use Magento\Framework\DB\Select;
 
@@ -45,8 +46,10 @@ public function getAllIds(FeedIndexMetadata $metadata): ?\Generator
         $connection = $this->resourceConnection->getConnection();
         $lastKnownId = -1;
         $continueReindex = true;
+        $dateTime = date_create();
+
         while ($continueReindex) {
-            $ids = $connection->fetchAll($this->getIdsSelect((int)$lastKnownId, $metadata));
+            $ids = $connection->fetchAll($this->getIdsSelect((int)$lastKnownId, $metadata, $dateTime));
             if (empty($ids)) {
                 $continueReindex = false;
             } else {
@@ -61,25 +64,37 @@ public function getAllIds(FeedIndexMetadata $metadata): ?\Generator
      *
      * @param int $lastKnownId
      * @param FeedIndexMetadata $metadata
+     * @param DateTime $dateTime
      * @return Select
      */
-    private function getIdsSelect(int $lastKnownId, FeedIndexMetadata $metadata): Select
+    private function getIdsSelect(int $lastKnownId, FeedIndexMetadata $metadata, DateTime $dateTime): Select
     {
         $connection = $this->resourceConnection->getConnection();
         $tableName = $this->resourceConnection->getTableName($metadata->getSourceTableName());
 
         $columnExpression = sprintf('s.%s', $metadata->getSourceTableIdentityField());
         $whereClause = sprintf('s.%s > ?', $metadata->getSourceTableIdentityField());
 
-        return $connection->select()
+        $select = $connection->select()
             ->from(
                 ['s' => $tableName],
                 [
                     $metadata->getFeedIdentity() => 's.' . $metadata->getSourceTableField(),
                     self::IDENTITY_FIELD_NAME => 's.' . $metadata->getSourceTableIdentityField()
                 ]
             )
-            ->where($whereClause, $lastKnownId)
+            ->where($whereClause, $lastKnownId);
+
+        if ($metadata->getFullReIndexSecondsLimit() != 0) {
+            $select->where(sprintf(
+                "s.%s >= DATE_SUB(STR_TO_DATE('%s', '%%Y-%%m-%%d %%H:%%i:%%s'), INTERVAL %d SECOND)",
+                $metadata->getSourceTableFieldOnFullReIndexLimit(),
+                $dateTime->format('Y-m-d H:i:s'),
+                $metadata->getFullReIndexSecondsLimit()
+            ));
+        }
+
+        return $select
             ->order($columnExpression)
             ->limit($metadata->getBatchSize());
     }

DataExporter/Model/Indexer/FeedIndexMetadata.php

@@ -59,6 +59,16 @@ class FeedIndexMetadata
      */
     private $sourceTableIdentityField;
 
+    /**
+     * @var int
+     */
+    private $fullReindexSecondsLimit;
+
+    /**
+     * @var string
+     */
+    private $sourceTableFieldOnFullReIndexLimit;
+
     /**
      * @param string $feedName
      * @param string $sourceTableName
@@ -69,6 +79,8 @@ class FeedIndexMetadata
      * @param array $feedTableMutableColumns
      * @param int $batchSize
      * @param string|null $sourceTableIdentityField
+     * @param int $fullReIndexSecondsLimit
+     * @param string $sourceTableFieldOnFullReIndexLimit
      */
     public function __construct(
         string $feedName,
@@ -79,7 +91,9 @@ public function __construct(
         string $feedTableField,
         array $feedTableMutableColumns,
         int $batchSize = 100,
-        string $sourceTableIdentityField = null
+        string $sourceTableIdentityField = null,
+        int $fullReIndexSecondsLimit = 0,
+        string $sourceTableFieldOnFullReIndexLimit = 'updated_at'
     ) {
         $this->feedName = $feedName;
         $this->sourceTableName = $sourceTableName;
@@ -90,6 +104,8 @@ public function __construct(
         $this->feedTableMutableColumns = $feedTableMutableColumns;
         $this->batchSize = $batchSize;
         $this->sourceTableIdentityField = $sourceTableIdentityField ?? $sourceTableField;
+        $this->fullReindexSecondsLimit = $fullReIndexSecondsLimit;
+        $this->sourceTableFieldOnFullReIndexLimit = $sourceTableFieldOnFullReIndexLimit;
     }
 
     /**
@@ -181,4 +197,22 @@ public function getFeedTableMutableColumns(): array
     {
         return $this->feedTableMutableColumns;
     }
+
+    /**
+     * Determines the amount of seconds back in time when triggering a full reindex
+     * @return int the amount in seconds, 0 means no limit
+     */
+    public function getFullReIndexSecondsLimit(): int
+    {
+        return $this->fullReindexSecondsLimit;
+    }
+
+    /**
+     * Table field name to use when full reindex is limited (see fullReindexSecondsLimit)
+     * @return string the field name
+     */
+    public function getSourceTableFieldOnFullReIndexLimit(): string
+    {
+        return $this->sourceTableFieldOnFullReIndexLimit;
+    }
 }

SalesOrdersDataExporter/README.md

@@ -30,7 +30,12 @@ Current time will be used by default
 ```
 
 ### Reindexing
-To export full orders data simply run reindex command:
+To export orders data simply run reindex command:
 ```shell
 bin/magento indexer:reindex sales_order_data_exporter_v2
 ```
+
+Reindexing is currently limited to orders modified in the last 7 days.
+
+Note that `commerce-data-export:orders:link` will assign uuids to the orders older than last 7 days but only orders 
+modified within that timeframe will be exported. 

SalesOrdersDataExporter/etc/di.xml

@@ -19,6 +19,8 @@
             <argument name="feedTableMutableColumns" xsi:type="array">
                 <item name="feed_data" xsi:type="string">feed_data</item>
             </argument>
+            <argument name="fullReIndexSecondsLimit" xsi:type="number">604800</argument> <!-- 7 days -->
+            <argument name="sourceTableFieldOnFullReIndexLimit" xsi:type="string">updated_at</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\SalesOrdersDataExporter\Model\Indexer\OrdersFeedIndexer" type="Magento\DataExporter\Model\Indexer\FeedIndexer">