Skip to content

Commit

Permalink
feat(resource-pool): Implement a lightweight resource pool
Browse files Browse the repository at this point in the history
  • Loading branch information
dborsatto committed Nov 28, 2018
1 parent a7aa610 commit 5f5d7c3
Show file tree
Hide file tree
Showing 18 changed files with 698 additions and 238 deletions.
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,17 @@ This project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased](https://github.com/contentful/contentful.php/compare/4.0.2...HEAD)

<!-- PENDING-CHANGES -->
## Added
* When working with huge datasets (tens of thousands of resources), for instance with custom CLI commands, the default resource pool may have caused the application to use too much memory and crash. Now there are two different resource pools: `Contentful\Delivery\ResourcePool\Standard`, which is a lightweight implemention and only caches space, environment and content types, and `Contentful\Delivery\ResourcePool\Extended`, which also keeps a reference to entries and assets. The latter class was known as `Contentful\Delivery\ResourcePool`, which has now been deprecated and will be removed in version 5.0. The `Extended` class is the default one, as it is designed to work with regular PHP requests. In order to use the `Standard` pool, you must enable it through the `ClienOptions` class:
```php
$options = ClientOptions::create()
->withLowMemoryResourcePool();
$client = new Client($token, $spaceId, $environmentId, $options);
```
As this resource pool provides no caching (and `include` values are now skipped, for this reason), it might cause the SDK to make more API calls, so its used is discouraged for regular web applications.
<!-- /PENDING-CHANGES -->

## [4.0.2](https://github.com/contentful/contentful.php/tree/4.0.2) (2018-11-09)

### Fixed
Expand Down
5 changes: 5 additions & 0 deletions UPGRADE-5.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# UPGRADE FROM 4.x to 5.0

## Removal of general ResourcePool class

The class `Contentful\Delivery\ResourcePool` was deprecated in version 4.1, and was removed in 5.0; if you were type hinting against this implementation, you can change it to use `Contentful\Delivery\ResourcePool\Extended` instead, or better yet, use the interface `Contentful\Core\Resource\ResourcePoolInterface`.
9 changes: 2 additions & 7 deletions src/Client.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
use Contentful\Delivery\Resource\Entry;
use Contentful\Delivery\Resource\Environment;
use Contentful\Delivery\Resource\Space;
use Contentful\Delivery\ResourcePool\Factory;
use Contentful\Delivery\Synchronization\Manager;
use Contentful\RichText\Parser;

Expand Down Expand Up @@ -134,13 +135,7 @@ public function __construct(
$this->isDeliveryApi = self::URI_PREVIEW !== $options->getHost();
$this->defaultLocale = $options->getDefaultLocale();

$this->resourcePool = new ResourcePool(
$this,
$options->getCacheItemPool(),
$options->hasCacheAutoWarmup(),
$options->hasCacheContent()
);

$this->resourcePool = Factory::create($this, $options);
$this->scopedJsonDecoder = new ScopedJsonDecoder($this->spaceId, $this->environmentId);
$this->linkResolver = new LinkResolver($this, $this->resourcePool);
$this->richTextParser = new Parser($this->linkResolver);
Expand Down
41 changes: 41 additions & 0 deletions src/ClientOptions.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ class ClientOptions
*/
private $httpClient;

/**
* @var bool
*/
private $usesLowMemoryResourcePool = \false;

/**
* ClientOptions constructor.
*/
Expand Down Expand Up @@ -220,4 +225,40 @@ public function getHttpClient(): HttpClient
{
return $this->httpClient;
}

/**
* Configures the client to use the default resource pool implementation,
* which may use more memory in extreme scenarios (tens of thousands of resources).
*
* @return ClientOptions
*/
public function withNormalResourcePool(): self
{
$this->usesLowMemoryResourcePool = \false;

return $this;
}

/**
* Configures the client to use a resource pool which will not cache entries and assets,
* which is useful when handling tens of thousand of resources,
* but it may cause extra API calls in normal scenarios.
* Use this option only if the default resource pool is causing you memory errors.
*
* @return ClientOptions
*/
public function withLowMemoryResourcePool(): self
{
$this->usesLowMemoryResourcePool = \true;

return $this;
}

/**
* @return bool
*/
public function usesLowMemoryResourcePool(): bool
{
return $this->usesLowMemoryResourcePool;
}
}
51 changes: 20 additions & 31 deletions src/Mapper/Entry.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
use Contentful\Core\Api\Link;
use Contentful\Core\Api\Location;
use Contentful\Delivery\Resource\ContentType as ResourceContentType;
use Contentful\Delivery\Resource\ContentType\Field as ResourceContentTypeField;
use Contentful\Delivery\Resource\Entry as ResourceClass;
use Contentful\Delivery\SystemProperties\Entry as SystemProperties;
use function GuzzleHttp\json_decode as guzzle_json_decode;
Expand All @@ -38,12 +37,24 @@ public function map($resource, array $data): ResourceClass
$sys = $this->createSystemProperties(SystemProperties::class, $data);
$locale = $sys->getLocale();

// We normalize the field data to always contain locales.
foreach ($data['fields'] ?? [] as $name => $value) {
// If the value is an empty array, and no locale was used,
// we remove the value as the entry itself will handle default values.
if (!$locale && $value === []) {
unset($data['fields'][$name]);
continue;
}

$data['fields'][$name] = $locale ? [$locale => $value] : $value;
}

/** @var ResourceClass $entry */
$entry = $this->hydrator->hydrate($resource ?: ResourceClass::class, [
'sys' => $sys,
'client' => $this->client,
'fields' => isset($data['fields'])
? $this->buildFields($sys->getContentType(), $data['fields'], $locale, $resource)
? $this->buildFields($sys->getContentType(), $data['fields'], $resource)
: [],
]);

Expand All @@ -55,27 +66,19 @@ public function map($resource, array $data): ResourceClass
/**
* @param ResourceContentType $contentType
* @param array $fields
* @param string|null $locale
* @param ResourceClass|null $previous
*
* @return array
*/
private function buildFields(
ResourceContentType $contentType,
array $fields,
string $locale = \null,
ResourceClass $previous = \null
): array {
// We normalize the field data to always contain locales.
foreach ($fields as $name => $data) {
$fields[$name] = $locale ? [$locale => $data] : $data;
}

if ($previous) {
$fields = $this->mergePreviousFields($fields, $previous);
}

$result = [];
foreach ($fields as $name => $data) {
$field = $contentType->getField($name);

Expand All @@ -94,14 +97,16 @@ private function buildFields(
$field = $contentType->addUnknownField($name);
}

// If the field is empty (has no values for locales) we simply skip it;
// the entry class will be able to properly return default values for those situations.
if ($data) {
$result[$name] = $this->buildField($field, $data);
foreach ($data as $locale => $value) {
$fields[$name][$locale] = $this->formatValue(
$field->getType(),
$value,
$field->getItemsType()
);
}
}

return $result;
return $fields;
}

/**
Expand Down Expand Up @@ -138,22 +143,6 @@ private function mergePreviousFields(array $fields, ResourceClass $entry): array
return $currentFields;
}

/**
* @param ResourceContentTypeField $field
* @param array $data
*
* @return array
*/
private function buildField(ResourceContentTypeField $field, array $data): array
{
$result = [];
foreach ($data as $locale => $value) {
$result[$locale] = $this->formatValue($field->getType(), $value, $field->getItemsType());
}

return $result;
}

/**
* Transforms values from the original JSON representation to an appropriate PHP representation.
*
Expand Down
2 changes: 1 addition & 1 deletion src/Mapper/ResourceArray.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* ResourceArray class.
*
* This class is responsible for converting raw API data into a PHP object
* of class Contentful\ResourceArray.
* of class Contentful\Core\Resource\ResourceArray.
*/
class ResourceArray extends BaseMapper
{
Expand Down
Loading

0 comments on commit 5f5d7c3

Please sign in to comment.