Merge pull request #91 from findologic/PLENTY-483-Imports-fail-due-to…

…-insufficient-memory Plenty-483-imports-fail-due-to-insufficient-memory
findologic · Nov 9, 2023 · 5b62533 · 5b62533
2 parents 0109205 + 07f2bae
commit 5b62533
Show file tree

Hide file tree

Showing 21 changed files with 68 additions and 32 deletions.
diff --git a/src/Config.php b/src/Config.php
@@ -47,6 +47,8 @@ class Config
 
     private bool $exportFreeTextFields = true;
 
+    private int $itemsPerPage = 100;
+
     private ?float $exportReferrerId = null;
 
     private string $exportDimensionUnit = 'mm';
@@ -97,6 +99,7 @@ public static function fromArray(array $data, bool $debug = false): self
             'exportReferrerId' => self::getFloatCastExportReferrerId($plentyConfig['export_referrer_id'] ?? null),
             'exportDimensionUnit' => $plentyConfig['export_dimension_unit'],
             'exportWeightUnit' => $plentyConfig['export_weight_unit'],
+            'itemsPerPage' => $plentyConfig['items_per_page'],
             'debug' => $debug
         ]);
     }
@@ -123,6 +126,7 @@ public static function fromEnvironment(): Config
             'exportReferrerId' => self::getFloatCastExportReferrerId(Utils::env('EXPORT_REFERRER_ID')),
             'exportDimensionUnit' => Utils::env('EXPORT_DIMENSION_UNIT'),
             'exportWeightUnit' => Utils::env('EXPORT_WEIGHT_UNIT'),
+            'itemsPerPage' => (int)Utils::env('ITEMS_PER_PAGE'),
             'debug' => (bool)Utils::env('DEBUG')
         ]);
     }
@@ -334,6 +338,16 @@ public function setExportFreeTextFields(bool $exportFreeTextFields): void
         $this->exportFreeTextFields = $exportFreeTextFields;
     }
 
+    public function getItemsPerPage(): int
+    {
+        return $this->itemsPerPage;
+    }
+
+    public function setItemsPerPage(int $itemsPerPage): void
+    {
+        $this->itemsPerPage = $itemsPerPage;
+    }
+
     private static function getFloatCastExportReferrerId($exportReferrerId): ?float
     {
         if (is_numeric($exportReferrerId)) {

diff --git a/src/Exporter/Exporter.php b/src/Exporter/Exporter.php
@@ -217,7 +217,7 @@ protected function exportProducts(): void
             $products = $this->getItems($page);
             $variations = $products->getAllIds() ? $this->getItemVariations($products->getAllIds()) : [];
             $this->wrapData(count($products->all()), $products, $variations, $propertySelection);
-            $this->offset += ItemVariationRequest::$ITEMS_PER_PAGE;
+            $this->offset += $this->itemRequest->getItemsPerPage();
 
             $page++;
         } while (!$products->isLastPage());
@@ -235,6 +235,7 @@ protected function exportProducts(): void
     private function getItems($page): ItemResponse
     {
         $this->itemRequest->setPage($page);
+        $this->itemRequest->setItemsPerPage($this->config->getItemsPerPage());
         $response = $this->client->send($this->itemRequest);
 
         return ItemParser::parse($response);
@@ -262,7 +263,7 @@ private function getItemVariations(array $itemIds): PimVariationResponse
             ->setParam('clientId', $this->registryService->getWebStore()->getStoreIdentifier())
             ->setWith($this->getRequiredVariationValues());
 
-        foreach (Utils::sendIterableRequest($this->client, $this->itemVariationRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $this->itemVariationRequest, $this->config) as $response) {
             $pimVariationResponse = PimVariationsParser::parse($response);
             $variations = array_merge($pimVariationResponse->all(), $variations);
         }

diff --git a/src/RegistryService.php b/src/RegistryService.php
@@ -365,7 +365,7 @@ protected function fetchCategories(): void
 
         $categoryRequest = new CategoryRequest($webStore->getStoreIdentifier());
 
-        foreach (Utils::sendIterableRequest($this->client, $categoryRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $categoryRequest, $this->config) as $response) {
             $categoryResponse = CategoryParser::parse($response);
             $categoriesMatchingCriteria = $categoryResponse->find([
                 'details' => [
@@ -393,7 +393,7 @@ protected function fetchCategories(): void
     protected function fetchVat(): void
     {
         $vatRequest = new VatRequest();
-        foreach (Utils::sendIterableRequest($this->client, $vatRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $vatRequest, $this->config) as $response) {
             $vatResponse = VatParser::parse($response);
 
             foreach ($vatResponse->all() as $vat) {
@@ -421,7 +421,7 @@ protected function fetchVat(): void
     protected function fetchSalesPrices(): void
     {
         $salesPriceRequest = new SalesPriceRequest();
-        foreach (Utils::sendIterableRequest($this->client, $salesPriceRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $salesPriceRequest, $this->config) as $response) {
             $salesPriceResponse = SalesPriceParser::parse($response);
 
             foreach ($salesPriceResponse->all() as $salesPrice) {
@@ -454,7 +454,7 @@ protected function fetchSalesPrices(): void
     protected function fetchAttributes(): void
     {
         $attributeRequest = new AttributeRequest();
-        foreach (Utils::sendIterableRequest($this->client, $attributeRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $attributeRequest, $this->config) as $response) {
             $attributeResponse = AttributeParser::parse($response);
 
             foreach ($attributeResponse->all() as $attribute) {
@@ -476,7 +476,7 @@ protected function fetchAttributes(): void
     protected function fetchManufacturers(): void
     {
         $manufacturerRequest = new ManufacturerRequest();
-        foreach (Utils::sendIterableRequest($this->client, $manufacturerRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $manufacturerRequest, $this->config) as $response) {
             $manufacturerResponse = ManufacturerParser::parse($response);
 
             foreach ($manufacturerResponse->all() as $manufacturer) {
@@ -499,7 +499,7 @@ protected function fetchProperties(): void
     {
         $propertyRequest = new PropertyRequest();
 
-        foreach (Utils::sendIterableRequest($this->client, $propertyRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $propertyRequest, $this->config) as $response) {
             $propertyResponse = PropertyParser::parse($response);
 
             foreach ($propertyResponse->all() as $property) {
@@ -525,7 +525,7 @@ protected function fetchPropertyGroups(): void
         try {
             $propertyGroupRequest = new PropertyGroupRequest();
 
-            foreach (Utils::sendIterableRequest($this->client, $propertyGroupRequest) as $response) {
+            foreach (Utils::sendIterableRequest($this->client, $propertyGroupRequest, $this->config) as $response) {
                 $propertyGroupResponse = PropertyGroupParser::parse($response);
 
                 foreach ($propertyGroupResponse->all() as $propertyGroup) {
@@ -555,7 +555,7 @@ protected function fetchItemProperties(): void
         $with = ['names', 'selections'];
         $propertyRequest = new ItemPropertyRequest($with);
 
-        foreach (Utils::sendIterableRequest($this->client, $propertyRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $propertyRequest, $this->config) as $response) {
             $propertyResponse = ItemPropertyParser::parse($response);
 
             foreach ($propertyResponse->all() as $property) {
@@ -577,7 +577,7 @@ protected function fetchItemProperties(): void
     protected function fetchUnits(): void
     {
         $unitRequest = new UnitRequest();
-        foreach (Utils::sendIterableRequest($this->client, $unitRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $unitRequest, $this->config) as $response) {
             $unitResponse = UnitParser::parse($response);
 
             foreach ($unitResponse->all() as $unit) {
@@ -601,7 +601,7 @@ protected function fetchPropertySelections(): void
             $selectionsRequest = new PropertySelectionRequest();
 
             $selections = [];
-            foreach (Utils::sendIterableRequest($this->client, $selectionsRequest) as $response) {
+            foreach (Utils::sendIterableRequest($this->client, $selectionsRequest, $this->config) as $response) {
                 $selectionsResponse = PropertySelectionParser::parse($response);
                 $selections = array_merge($selectionsResponse->all(), $selections);
             }
@@ -632,7 +632,7 @@ protected function fetchItemPropertyGroups(): void
     {
         $propertyGroupRequest = new ItemPropertyGroupRequest('names');
 
-        foreach (Utils::sendIterableRequest($this->client, $propertyGroupRequest) as $response) {
+        foreach (Utils::sendIterableRequest($this->client, $propertyGroupRequest, $this->config) as $response) {
             $propertyGroupResponse = ItemPropertyGroupParser::parse($response);
 
             foreach ($propertyGroupResponse->all() as $propertyGroup) {

diff --git a/src/Request/AttributeRequest.php b/src/Request/AttributeRequest.php
@@ -15,7 +15,7 @@ public function __construct(?string $with = 'names')
             'items/attributes',
             [
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE,
+                'itemsPerPage' => $this->itemsPerPage,
                 'with' => $with
             ]
         );

diff --git a/src/Request/CategoryRequest.php b/src/Request/CategoryRequest.php
@@ -18,7 +18,7 @@ public function __construct(int $storeIdentifier)
                 'with' => ['details'],
                 'plentyId' => $storeIdentifier,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/ItemPropertyGroupRequest.php b/src/Request/ItemPropertyGroupRequest.php
@@ -16,7 +16,7 @@ public function __construct(string $with)
             [
                 'with' => $with,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/ItemPropertyRequest.php b/src/Request/ItemPropertyRequest.php
@@ -18,7 +18,7 @@ public function __construct(?array $with = null, ?string $updatedAt = null, ?str
                 'updatedAt' => $updatedAt,
                 'groupId' => $groupId,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/ItemRequest.php b/src/Request/ItemRequest.php
@@ -37,7 +37,7 @@ public function __construct(
                 'variationRelatedUpdatedBetween' => $variationRelatedUpdatedBetween,
                 'or' => $or,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/IterableRequest.php b/src/Request/IterableRequest.php
@@ -6,15 +6,14 @@
 
 trait IterableRequest
 {
-    /** Maximum count of entities per page. */
-    public static int $ITEMS_PER_PAGE = 100;
     protected int $page = 1;
+    protected int $itemsPerPage = 100;
 
     public function getParams(): array
     {
         $params = parent::getParams();
         $params['page'] = $this->page;
-        $params['itemsPerPage'] = static::$ITEMS_PER_PAGE;
+        $params['itemsPerPage'] = $this->itemsPerPage;
 
         return $params;
     }
@@ -31,9 +30,14 @@ public function getPage(): int
         return $this->page;
     }
 
+    public function getItemsPerPage(): int
+    {
+        return $this->itemsPerPage;
+    }
+
     public function setItemsPerPage(int $itemsPerPage): self
     {
-        static::$ITEMS_PER_PAGE = $itemsPerPage;
+        $this->itemsPerPage = $itemsPerPage;
 
         return $this;
     }

diff --git a/src/Request/IterableRequestInterface.php b/src/Request/IterableRequestInterface.php
@@ -11,4 +11,8 @@ public function getParams(): array;
     public function setPage(int $page);
 
     public function getPage(): int;
+
+    public function setItemsPerPage(int $itemsPerPage);
+
+    public function getItemsPerPage(): int;
 }
diff --git a/src/Request/ManufacturerRequest.php b/src/Request/ManufacturerRequest.php
@@ -18,7 +18,7 @@ public function __construct(?string $with = null, ?string $updatedAt = null, ?st
                 'updatedAt' => $updatedAt,
                 'name' => $name,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/PropertyGroupRequest.php b/src/Request/PropertyGroupRequest.php
@@ -15,7 +15,7 @@ public function __construct()
             'v2/properties/groups',
             [
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE,
+                'itemsPerPage' => $this->itemsPerPage,
                 'with' => 'names'
             ]
         );

diff --git a/src/Request/PropertyRequest.php b/src/Request/PropertyRequest.php
@@ -15,7 +15,7 @@ public function __construct()
             'v2/properties',
             [
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE,
+                'itemsPerPage' => $this->itemsPerPage,
                 'with' => 'names,amazon,options,groups'
             ]
         );

diff --git a/src/Request/PropertySelectionRequest.php b/src/Request/PropertySelectionRequest.php
@@ -15,7 +15,7 @@ public function __construct()
             'properties/selections',
             [
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/SalesPriceRequest.php b/src/Request/SalesPriceRequest.php
@@ -16,7 +16,7 @@ public function __construct(?string $updatedAt = null)
             [
                 'updatedAt' => $updatedAt,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/UnitRequest.php b/src/Request/UnitRequest.php
@@ -16,7 +16,7 @@ public function __construct(?string $updatedAt = null)
             [
                 'updatedAt' => $updatedAt,
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Request/VatRequest.php b/src/Request/VatRequest.php
@@ -15,7 +15,7 @@ public function __construct()
             'vat',
             [
                 'page' => $this->page,
-                'itemsPerPage' => self::$ITEMS_PER_PAGE
+                'itemsPerPage' => $this->itemsPerPage
             ]
         );
     }

diff --git a/src/Utils.php b/src/Utils.php
@@ -14,6 +14,7 @@
 use FINDOLOGIC\PlentyMarketsRestExporter\Exception\ThrottlingException;
 use FINDOLOGIC\PlentyMarketsRestExporter\Request\IterableRequestInterface;
 use FINDOLOGIC\PlentyMarketsRestExporter\Request\Request;
+use FINDOLOGIC\PlentyMarketsRestExporter\Config;
 use GuzzleHttp\Client as GuzzleClient;
 use GuzzleHttp\Exception\GuzzleException;
 use InvalidArgumentException;
@@ -38,7 +39,7 @@ final class Utils
      * @throws CriticalException
      *
      */
-    public static function sendIterableRequest(Client $client, Request $request): array
+    public static function sendIterableRequest(Client $client, Request $request, Config $config): array
     {
         if (!$request instanceof IterableRequestInterface) {
             throw new InvalidArgumentException(sprintf(
@@ -49,6 +50,8 @@ public static function sendIterableRequest(Client $client, Request $request): ar
 
         $responses = [];
         $lastPage = false;
+        $request->setItemsPerPage($config->getItemsPerPage());
+
         while (!$lastPage) {
             $response = $client->send($request);
             $lastPage = self::parseIsLastPage($response);

diff --git a/tests/ConfigTest.php b/tests/ConfigTest.php
@@ -67,6 +67,7 @@ public function testDataCanBeFetchedFromCustomerLoginData(): void
         $expectedDimensionUnit = 'm';
         $expectedWeightUnit = 'kg';
         $expectedUseVariants = false;
+        $expectedItemsPerPage = 100;
 
         $accountResponse = [
             '1234' => [
@@ -93,6 +94,7 @@ public function testDataCanBeFetchedFromCustomerLoginData(): void
                     'export_free_text_fields' => $expectedExportFreeTextFields,
                     'export_dimension_unit' => $expectedDimensionUnit,
                     'export_weight_unit' => $expectedWeightUnit,
+                    'items_per_page' => $expectedItemsPerPage
                 ],
             ]
         ];
@@ -117,6 +119,7 @@ public function testDataCanBeFetchedFromCustomerLoginData(): void
         $this->assertSame($expectedDimensionUnit, $config->getExportDimensionUnit());
         $this->assertSame($expectedWeightUnit, $config->getExportWeightUnit());
         $this->assertSame($expectedUseVariants, $config->getUseVariants());
+        $this->assertSame($expectedItemsPerPage, $config->getItemsPerPage());
     }
 
     /**

diff --git a/tests/MockData/AccountResponse/response.json b/tests/MockData/AccountResponse/response.json
@@ -99,7 +99,8 @@
       "export_ordernumber_variant_barcodes": true,
       "export_free_text_fields": true,
       "export_dimension_unit": "mm",
-      "export_weight_unit": "g"
+      "export_weight_unit": "g",
+      "items_per_page": 100
     },
     "synonyms": "",
     "bonus": "{}",

diff --git a/tests/UtilsTest.php b/tests/UtilsTest.php
@@ -250,10 +250,16 @@ public function testSendIterableRequestFailsForNonIterableRequests(): void
         ));
 
         $nonIterableRequest = new PluginConfigurationRequest(1234, 1234);
+
+        $config = Utils::getExportConfiguration(
+            null,
+            $this->clientMock
+        );
+
         $client = $this->getMockBuilder(PlentyRestClient::class)
             ->disableOriginalConstructor()
             ->getMock();
 
-        Utils::sendIterableRequest($client, $nonIterableRequest);
+        Utils::sendIterableRequest($client, $nonIterableRequest, $config);
     }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,7 +15,7 @@ public function __construct() @@
                 'vat',
                 [
                     'page' => $this->page,
-                    'itemsPerPage' => self::$ITEMS_PER_PAGE
+                    'itemsPerPage' => $this->itemsPerPage
                 ]
             );
         }
@@ Expand Down @@