Merge pull request #258 from contentful/feat/low-memory-resource-pool

Implement a lightweight resource pool
contentful · Nov 28, 2018 · 86f311b · 86f311b
2 parents a7aa610 + 5f5d7c3
commit 86f311b
Show file tree

Hide file tree

Showing 18 changed files with 698 additions and 238 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,17 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased](https://github.com/contentful/contentful.php/compare/4.0.2...HEAD)
 
+<!-- PENDING-CHANGES -->
+## Added
+* When working with huge datasets (tens of thousands of resources), for instance with custom CLI commands, the default resource pool may have caused the application to use too much memory and crash. Now there are two different resource pools: `Contentful\Delivery\ResourcePool\Standard`, which is a lightweight implemention and only caches space, environment and content types, and `Contentful\Delivery\ResourcePool\Extended`, which also keeps a reference to entries and assets. The latter class was known as `Contentful\Delivery\ResourcePool`, which has now been deprecated and will be removed in version 5.0. The `Extended` class is the default one, as it is designed to work with regular PHP requests. In order to use the `Standard` pool, you must enable it through the `ClienOptions` class:
+  ```php
+  $options = ClientOptions::create()
+      ->withLowMemoryResourcePool();
+  $client = new Client($token, $spaceId, $environmentId, $options);
+  ```
+  As this resource pool provides no caching (and `include` values are now skipped, for this reason), it might cause the SDK to make more API calls, so its used is discouraged for regular web applications.
+<!-- /PENDING-CHANGES -->
+
 ## [4.0.2](https://github.com/contentful/contentful.php/tree/4.0.2) (2018-11-09)
 
 ### Fixed

diff --git a/UPGRADE-5.0.md b/UPGRADE-5.0.md
@@ -0,0 +1,5 @@
+# UPGRADE FROM 4.x to 5.0
+
+## Removal of general ResourcePool class
+
+The class `Contentful\Delivery\ResourcePool` was deprecated in version 4.1, and was removed in 5.0; if you were type hinting against this implementation, you can change it to use `Contentful\Delivery\ResourcePool\Extended` instead, or better yet, use the interface `Contentful\Core\Resource\ResourcePoolInterface`.
diff --git a/src/Client.php b/src/Client.php
@@ -26,6 +26,7 @@
 use Contentful\Delivery\Resource\Entry;
 use Contentful\Delivery\Resource\Environment;
 use Contentful\Delivery\Resource\Space;
+use Contentful\Delivery\ResourcePool\Factory;
 use Contentful\Delivery\Synchronization\Manager;
 use Contentful\RichText\Parser;
 
@@ -134,13 +135,7 @@ public function __construct(
         $this->isDeliveryApi = self::URI_PREVIEW !== $options->getHost();
         $this->defaultLocale = $options->getDefaultLocale();
 
-        $this->resourcePool = new ResourcePool(
-            $this,
-            $options->getCacheItemPool(),
-            $options->hasCacheAutoWarmup(),
-            $options->hasCacheContent()
-        );
-
+        $this->resourcePool = Factory::create($this, $options);
         $this->scopedJsonDecoder = new ScopedJsonDecoder($this->spaceId, $this->environmentId);
         $this->linkResolver = new LinkResolver($this, $this->resourcePool);
         $this->richTextParser = new Parser($this->linkResolver);

diff --git a/src/ClientOptions.php b/src/ClientOptions.php
@@ -54,6 +54,11 @@ class ClientOptions
      */
     private $httpClient;
 
+    /**
+     * @var bool
+     */
+    private $usesLowMemoryResourcePool = \false;
+
     /**
      * ClientOptions constructor.
      */
@@ -220,4 +225,40 @@ public function getHttpClient(): HttpClient
     {
         return $this->httpClient;
     }
+
+    /**
+     * Configures the client to use the default resource pool implementation,
+     * which may use more memory in extreme scenarios (tens of thousands of resources).
+     *
+     * @return ClientOptions
+     */
+    public function withNormalResourcePool(): self
+    {
+        $this->usesLowMemoryResourcePool = \false;
+
+        return $this;
+    }
+
+    /**
+     * Configures the client to use a resource pool which will not cache entries and assets,
+     * which is useful when handling tens of thousand of resources,
+     * but it may cause extra API calls in normal scenarios.
+     * Use this option only if the default resource pool is causing you memory errors.
+     *
+     * @return ClientOptions
+     */
+    public function withLowMemoryResourcePool(): self
+    {
+        $this->usesLowMemoryResourcePool = \true;
+
+        return $this;
+    }
+
+    /**
+     * @return bool
+     */
+    public function usesLowMemoryResourcePool(): bool
+    {
+        return $this->usesLowMemoryResourcePool;
+    }
 }
diff --git a/src/Mapper/Entry.php b/src/Mapper/Entry.php
@@ -15,7 +15,6 @@
 use Contentful\Core\Api\Link;
 use Contentful\Core\Api\Location;
 use Contentful\Delivery\Resource\ContentType as ResourceContentType;
-use Contentful\Delivery\Resource\ContentType\Field as ResourceContentTypeField;
 use Contentful\Delivery\Resource\Entry as ResourceClass;
 use Contentful\Delivery\SystemProperties\Entry as SystemProperties;
 use function GuzzleHttp\json_decode as guzzle_json_decode;
@@ -38,12 +37,24 @@ public function map($resource, array $data): ResourceClass
         $sys = $this->createSystemProperties(SystemProperties::class, $data);
         $locale = $sys->getLocale();
 
+        // We normalize the field data to always contain locales.
+        foreach ($data['fields'] ?? [] as $name => $value) {
+            // If the value is an empty array, and no locale was used,
+            // we remove the value as the entry itself will handle default values.
+            if (!$locale && $value === []) {
+                unset($data['fields'][$name]);
+                continue;
+            }
+
+            $data['fields'][$name] = $locale ? [$locale => $value] : $value;
+        }
+
         /** @var ResourceClass $entry */
         $entry = $this->hydrator->hydrate($resource ?: ResourceClass::class, [
             'sys' => $sys,
             'client' => $this->client,
             'fields' => isset($data['fields'])
-                ? $this->buildFields($sys->getContentType(), $data['fields'], $locale, $resource)
+                ? $this->buildFields($sys->getContentType(), $data['fields'], $resource)
                 : [],
         ]);
 
@@ -55,27 +66,19 @@ public function map($resource, array $data): ResourceClass
     /**
      * @param ResourceContentType $contentType
      * @param array               $fields
-     * @param string|null         $locale
      * @param ResourceClass|null  $previous
      *
      * @return array
      */
     private function buildFields(
         ResourceContentType $contentType,
         array $fields,
-        string $locale = \null,
         ResourceClass $previous = \null
     ): array {
-        // We normalize the field data to always contain locales.
-        foreach ($fields as $name => $data) {
-            $fields[$name] = $locale ? [$locale => $data] : $data;
-        }
-
         if ($previous) {
             $fields = $this->mergePreviousFields($fields, $previous);
         }
 
-        $result = [];
         foreach ($fields as $name => $data) {
             $field = $contentType->getField($name);
 
@@ -94,14 +97,16 @@ private function buildFields(
                 $field = $contentType->addUnknownField($name);
             }
 
-            // If the field is empty (has no values for locales) we simply skip it;
-            // the entry class will be able to properly return default values for those situations.
-            if ($data) {
-                $result[$name] = $this->buildField($field, $data);
+            foreach ($data as $locale => $value) {
+                $fields[$name][$locale] = $this->formatValue(
+                    $field->getType(),
+                    $value,
+                    $field->getItemsType()
+                );
             }
         }
 
-        return $result;
+        return $fields;
     }
 
     /**
@@ -138,22 +143,6 @@ private function mergePreviousFields(array $fields, ResourceClass $entry): array
         return $currentFields;
     }
 
-    /**
-     * @param ResourceContentTypeField $field
-     * @param array                    $data
-     *
-     * @return array
-     */
-    private function buildField(ResourceContentTypeField $field, array $data): array
-    {
-        $result = [];
-        foreach ($data as $locale => $value) {
-            $result[$locale] = $this->formatValue($field->getType(), $value, $field->getItemsType());
-        }
-
-        return $result;
-    }
-
     /**
      * Transforms values from the original JSON representation to an appropriate PHP representation.
      *

diff --git a/src/Mapper/ResourceArray.php b/src/Mapper/ResourceArray.php
@@ -17,7 +17,7 @@
  * ResourceArray class.
  *
  * This class is responsible for converting raw API data into a PHP object
- * of class Contentful\ResourceArray.
+ * of class Contentful\Core\Resource\ResourceArray.
  */
 class ResourceArray extends BaseMapper
 {