From fd08c1b9e1c038570175ff68e1a00851dfedd617 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 1 Jun 2016 19:00:50 +0200 Subject: [PATCH] Initial commit --- .drone.yml | 24 ++++ .gitignore | 4 + README.md | 24 ++++ Robots/Checker/RequestChecker.php | 56 ++++++++ Robots/Checker/RequestCheckerInterface.php | 29 ++++ Robots/Exception/RuleNotFoundException.php | 16 +++ Robots/Model/Rule.php | 131 ++++++++++++++++++ Robots/Model/RuleInterface.php | 53 +++++++ Robots/Provider/ArrayRuleProvider.php | 40 ++++++ Robots/Provider/CachedRuleProvider.php | 63 +++++++++ Robots/Provider/FallbackRuleProvider.php | 43 ++++++ Robots/Provider/RepositoryRuleProvider.php | 32 +++++ Robots/Provider/RuleProviderInterface.php | 21 +++ Robots/Repository/RuleRepositoryInterface.php | 18 +++ Robots/Resolver/TagResolver.php | 60 ++++++++ Robots/Resolver/TagResolverInterface.php | 20 +++ RobotsBundle/Command/InitializeCommand.php | 38 +++++ RobotsBundle/DagRobotsBundle.php | 36 +++++ .../DependencyInjection/Configuration.php | 125 +++++++++++++++++ .../DagRobotsExtension.php | 57 ++++++++ RobotsBundle/Doctrine/ORM/RuleRepository.php | 13 ++ RobotsBundle/Doctrine/RuleInitializer.php | 71 ++++++++++ .../EventListener/KernelResponseListener.php | 50 +++++++ RobotsBundle/Form/Type/RuleType.php | 29 ++++ RobotsBundle/Form/Type/TagChoiceType.php | 43 ++++++ .../config/doctrine/model/Rule.orm.xml | 26 ++++ RobotsBundle/Resources/config/services.xml | 60 ++++++++ composer.json | 35 +++++ phpspec.yml.dist | 12 ++ 29 files changed, 1229 insertions(+) create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 Robots/Checker/RequestChecker.php create mode 100644 Robots/Checker/RequestCheckerInterface.php create mode 100644 Robots/Exception/RuleNotFoundException.php create mode 100644 Robots/Model/Rule.php create mode 100644 Robots/Model/RuleInterface.php create mode 100644 Robots/Provider/ArrayRuleProvider.php create mode 100644 Robots/Provider/CachedRuleProvider.php create mode 100644 Robots/Provider/FallbackRuleProvider.php create mode 100644 Robots/Provider/RepositoryRuleProvider.php create mode 100644 Robots/Provider/RuleProviderInterface.php create mode 100644 Robots/Repository/RuleRepositoryInterface.php create mode 100644 Robots/Resolver/TagResolver.php create mode 100644 Robots/Resolver/TagResolverInterface.php create mode 100644 RobotsBundle/Command/InitializeCommand.php create mode 100644 RobotsBundle/DagRobotsBundle.php create mode 100644 RobotsBundle/DependencyInjection/Configuration.php create mode 100644 RobotsBundle/DependencyInjection/DagRobotsExtension.php create mode 100644 RobotsBundle/Doctrine/ORM/RuleRepository.php create mode 100644 RobotsBundle/Doctrine/RuleInitializer.php create mode 100644 RobotsBundle/EventListener/KernelResponseListener.php create mode 100644 RobotsBundle/Form/Type/RuleType.php create mode 100644 RobotsBundle/Form/Type/TagChoiceType.php create mode 100644 RobotsBundle/Resources/config/doctrine/model/Rule.orm.xml create mode 100644 RobotsBundle/Resources/config/services.xml create mode 100644 composer.json create mode 100644 phpspec.yml.dist diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..77ac4bd --- /dev/null +++ b/.drone.yml @@ -0,0 +1,24 @@ +cache: + mount: + - .git + - /drone/composer-cache + +build: + image: worldia/php:cli + environment: + COMPOSER_CACHE_DIR: /drone/composer-cache + commands: + - composer install --prefer-dist --no-interaction --no-progress --ansi + - bin/phpspec run + +notify: + slack_blame: + token: xoxp-2941314706-2941314708-20197447201-1ced90fcf4 + success: + username: "Bob" + icon: ":bob:" + message: "The build is fixed!" + failure: + username: "Bob" + icon: ":Bob:" + message: "The build is broken!" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..74df7dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +vendor/ +bin/ + +composer.lock \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..929b6d7 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +## RobotsBundle + +Add contextual robot meta headers. + +### Installation + +```sh +$ composer require cdaguerre/robots-bundle +``` + +### Configuration + +```yml +dag_robots: + rules: + - { route: 'homepage', tags: ['noindex', 'nofollow'], hosts: ['www.example.com'] } + - { route: 'product_show', tags: ['noindex', 'nofollow'], hosts: ['www.example1.com', 'www.example2.com'] } + - { route: 'category_show', tags: ['noindex', 'nofollow'] } +``` + +### TODO + +- [ ] Add route to serve robots.txt files dynamically. +- [ ] Add tests \ No newline at end of file diff --git a/Robots/Checker/RequestChecker.php b/Robots/Checker/RequestChecker.php new file mode 100644 index 0000000..51d6f81 --- /dev/null +++ b/Robots/Checker/RequestChecker.php @@ -0,0 +1,56 @@ + + */ +class RequestChecker implements RequestCheckerInterface +{ + /** + * @var array + */ + protected $crawlerUserAgents = array( + 'googlebot', + 'bingbot', + 'msnbot', + 'twitterbot', + 'yahoo', + 'baiduspider', + 'facebookexternalhit', + 'askjeeves', + 'teomabar', + ); + + /** + * @param array $crawlerUserAgents Additionnal user agents. + */ + public function __construct(array $crawlerUserAgents = array()) + { + $this->crawlerUserAgents = array_flip(array_merge($this->crawlerUserAgents, $crawlerUserAgents)); + } + + /** + * {@inheritdoc} + */ + public function isRobot(Request $request) + { + $userAgent = strtolower($request->headers->get('User-Agent')); + + return isset($this->crawlerUserAgents[$userAgent]); + } + + /** + * {@inheritdoc} + */ + public function isCrawler(Request $request) + { + if (null !== $request->query->get('_escaped_fragment_')) { + return true; + } + + return $this->isRobot($request); + } +} diff --git a/Robots/Checker/RequestCheckerInterface.php b/Robots/Checker/RequestCheckerInterface.php new file mode 100644 index 0000000..1cad0ce --- /dev/null +++ b/Robots/Checker/RequestCheckerInterface.php @@ -0,0 +1,29 @@ + + */ +interface RequestCheckerInterface +{ + /** + * Check if the given request was made by a robot. + * + * @param Request $request + * + * @return bool + */ + public function isRobot(Request $request); + + /** + * Check if the given request was made by a crawler. + * + * @param Request $request + * + * @return bool + */ + public function isCrawler(Request $request); +} diff --git a/Robots/Exception/RuleNotFoundException.php b/Robots/Exception/RuleNotFoundException.php new file mode 100644 index 0000000..97cb5cd --- /dev/null +++ b/Robots/Exception/RuleNotFoundException.php @@ -0,0 +1,16 @@ + + */ +class RuleNotFoundException extends \Exception +{ + public function __construct($route, $host) + { + parent::__construct(sprintf('There is no rule for host "%s" and route "%s".', $route, $host)); + } +} diff --git a/Robots/Model/Rule.php b/Robots/Model/Rule.php new file mode 100644 index 0000000..078b509 --- /dev/null +++ b/Robots/Model/Rule.php @@ -0,0 +1,131 @@ + + */ +class Rule implements RuleInterface +{ + /** + * @var int + */ + protected $id; + + /** + * @var string + */ + protected $route; + + /** + * @var string + */ + protected $host; + + /** + * @var array + */ + protected $tags = array(); + + /** + * @var \DateTime + */ + protected $createdAt; + + /** + * @var \DateTime + */ + protected $updatedAt; + + /** + * Constructor. + */ + public function __construct() + { + $this->createdAt = new \DateTime(); + } + + /** + * {@inheritdoc} + */ + public function getRoute() + { + return $this->route; + } + + /** + * {@inheritdoc} + */ + public function setRoute($route) + { + $this->route = $route; + } + + /** + * {@inheritdoc} + */ + public function getHost() + { + return $this->host; + } + + /** + * {@inheritdoc} + */ + public function setHost($host) + { + $this->host = $host; + } + + /** + * {@inheritdoc} + */ + public function getTags() + { + return $this->tags; + } + + /** + * {@inheritdoc} + */ + public function setTags(array $tags) + { + $this->tags = $tags; + } + + /** + * {@inheritdoc} + */ + public function getCreatedAt() + { + return $this->createdAt; + } + + /** + * {@inheritdoc} + */ + public function setCreatedAt(\DateTime $createdAt) + { + $this->createdAt = $createdAt; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getUpdatedAt() + { + return $this->updatedAt; + } + + /** + * {@inheritdoc} + */ + public function setUpdatedAt(\DateTime $updatedAt) + { + $this->updatedAt = $updatedAt; + + return $this; + } +} diff --git a/Robots/Model/RuleInterface.php b/Robots/Model/RuleInterface.php new file mode 100644 index 0000000..c24292e --- /dev/null +++ b/Robots/Model/RuleInterface.php @@ -0,0 +1,53 @@ + + */ +interface RuleInterface extends TimestampableInterface +{ + /** + * Get route. + * + * @return string + */ + public function getRoute(); + + /** + * Set route. + * + * @param string $route + */ + public function setRoute($route); + + /** + * Get host. + * + * @return string + */ + public function getHost(); + + /** + * Set host. + * + * @param string $host + */ + public function setHost($host); + + /** + * Get tags. + * + * @return array + */ + public function getTags(); + + /** + * Set tags. + * + * @param string $tags + */ + public function setTags(array $tags); +} diff --git a/Robots/Provider/ArrayRuleProvider.php b/Robots/Provider/ArrayRuleProvider.php new file mode 100644 index 0000000..697966a --- /dev/null +++ b/Robots/Provider/ArrayRuleProvider.php @@ -0,0 +1,40 @@ + + */ +class ArrayRuleProvider implements RuleProviderInterface +{ + /** + * @var array + */ + protected $rules; + + /** + * @param array $rules + */ + public function __construct(array $rules) + { + $this->rules = $rules; + } + + /** + * {@inheritdoc} + */ + public function findMatching($route, $host) + { + $rule = new Rule(); + $rule->setHost($host); + $rule->setRoute($route); + + if (isset($this->rules[$route][$host])) { + $rule->setTags($this->rules[$route][$host]); + } + + return $rule; + } +} diff --git a/Robots/Provider/CachedRuleProvider.php b/Robots/Provider/CachedRuleProvider.php new file mode 100644 index 0000000..bb01321 --- /dev/null +++ b/Robots/Provider/CachedRuleProvider.php @@ -0,0 +1,63 @@ + + */ +class CachedRuleProvider implements RuleProviderInterface +{ + const TTL = 60; + + /** + * @var Cache + */ + protected $cache; + + /** + * @var RuleProviderInterface + */ + protected $ruleProvider; + + /** + * @var int + */ + protected $ttl; + + /** + * @param RuleProviderInterface $ruleProvider + * @param Cache $cache + * @param int $ttl + */ + public function __construct(RuleProviderInterface $ruleProvider, Cache $cache, $ttl = self::TTL) + { + $this->ruleProvider = $ruleProvider; + $this->cache = $cache; + $this->ttl = $ttl; + } + + /** + * {@inheritdoc} + */ + public function findMatching($route, $host) + { + if ($this->cache->contains($this->getCacheKey())) { + return $this->cache->fetch($this->getCacheKey()); + } + + $rules = $this->ruleProvider->getRules(); + $this->cache->save($this->getCacheKey(), $rules, $this->ttl); + + return $rules; + } + + /** + * @return string + */ + private function getCacheKey() + { + return 'robot_rules'; + } +} diff --git a/Robots/Provider/FallbackRuleProvider.php b/Robots/Provider/FallbackRuleProvider.php new file mode 100644 index 0000000..6e0cc4a --- /dev/null +++ b/Robots/Provider/FallbackRuleProvider.php @@ -0,0 +1,43 @@ + + */ +class FallbackRuleProvider implements RuleProviderInterface +{ + /** + * @var RuleProviderInterface + */ + protected $defaultRuleProvider; + + /** + * @var RuleProviderInterface + */ + protected $fallbackRuleProvider; + + /** + * @param RuleProviderInterface $defaultRuleProvider + * @param RuleProviderInterface $fallbackRuleProvider + */ + public function __construct( + RuleProviderInterface $defaultRuleProvider, + RuleProviderInterface $fallbackRuleProvider + ) { + $this->defaultRuleProvider = $defaultRuleProvider; + $this->fallbackRuleProvider = $fallbackRuleProvider; + } + + /** + * {@inheritdoc} + */ + public function findMatching($route, $host) + { + try { + return $this->defaultRuleProvider->findMatching($route, $host); + } catch (\Exception $e) { + return $this->fallbackRuleProvider->findMatching($route, $host); + } + } +} diff --git a/Robots/Provider/RepositoryRuleProvider.php b/Robots/Provider/RepositoryRuleProvider.php new file mode 100644 index 0000000..1561e89 --- /dev/null +++ b/Robots/Provider/RepositoryRuleProvider.php @@ -0,0 +1,32 @@ + + */ +class RepositoryRuleProvider implements RuleProviderInterface +{ + /** + * @var RepositoryInterface + */ + protected $ruleRepository; + + /** + * @param RepositoryInterface $ruleRepository + */ + public function __construct(RepositoryInterface $ruleRepository) + { + $this->ruleRepository = $ruleRepository; + } + + /** + * {@inheritdoc} + */ + public function findMatching($route, $host) + { + return $this->ruleRepository->findOneBy(array('route' => $route, 'host' => $host)); + } +} diff --git a/Robots/Provider/RuleProviderInterface.php b/Robots/Provider/RuleProviderInterface.php new file mode 100644 index 0000000..cf2e2c5 --- /dev/null +++ b/Robots/Provider/RuleProviderInterface.php @@ -0,0 +1,21 @@ + + */ +interface RuleProviderInterface +{ + /** + * Find rule matching the given route and host. + * + * @param string $route + * @param string $host + * + * @return RuleInterface + */ + public function findMatching($route, $host); +} diff --git a/Robots/Repository/RuleRepositoryInterface.php b/Robots/Repository/RuleRepositoryInterface.php new file mode 100644 index 0000000..7ed24c5 --- /dev/null +++ b/Robots/Repository/RuleRepositoryInterface.php @@ -0,0 +1,18 @@ + + */ +interface RuleRepositoryInterface +{ + /** + * Retrieve all rules. + * + * @return array|RuleInterface[] + */ + public function findAll(); +} diff --git a/Robots/Resolver/TagResolver.php b/Robots/Resolver/TagResolver.php new file mode 100644 index 0000000..3e07405 --- /dev/null +++ b/Robots/Resolver/TagResolver.php @@ -0,0 +1,60 @@ + + */ +class TagResolver implements TagResolverInterface +{ + /** + * @var RuleProviderInterface + */ + protected $ruleProvider; + + /** + * @param RuleProviderInterface $ruleProvider + */ + public function __construct(RuleProviderInterface $ruleProvider) + { + $this->ruleProvider = $ruleProvider; + } + + /** + * {@inheritdoc} + */ + public function resolve(Request $request) + { + $host = $this->extractHost($request); + $route = $this->extractRoute($request); + + try { + return $this->ruleProvider->findMatching($route, $host)->getTags(); + } catch (RuleNotFoundException $e) { + return array(); + } + } + + /** + * @param Request $request + * + * @return string + */ + protected function extractHost(Request $request) + { + return $request->getHost(); + } + + /** + * @param Request $request + * + * @return string + */ + protected function extractRoute(Request $request) + { + return $request->get('_route'); + } +} diff --git a/Robots/Resolver/TagResolverInterface.php b/Robots/Resolver/TagResolverInterface.php new file mode 100644 index 0000000..55742f4 --- /dev/null +++ b/Robots/Resolver/TagResolverInterface.php @@ -0,0 +1,20 @@ + + */ +interface TagResolverInterface +{ + /** + * Resolve robot tags for the given request;. + * + * @param Request $request + * + * @return string|array + */ + public function resolve(Request $request); +} diff --git a/RobotsBundle/Command/InitializeCommand.php b/RobotsBundle/Command/InitializeCommand.php new file mode 100644 index 0000000..89728ac --- /dev/null +++ b/RobotsBundle/Command/InitializeCommand.php @@ -0,0 +1,38 @@ + + */ +class InitializeCommand extends ContainerAwareCommand +{ + /** + * {@inheritdoc} + */ + protected function configure() + { + $this + ->setName('dag:robots:initialize') + ->setDescription('Initialize default rules in the application.') + ->setHelp('The %command.name% command initializes default robot rules setup.') + ; + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + $output->writeln('Initializing default robot rules.'); + + $initializer = $this->getContainer()->get('dag.robots.rule_initializer'); + $initializer->initialize($output); + + $output->writeln('Completed!'); + } +} diff --git a/RobotsBundle/DagRobotsBundle.php b/RobotsBundle/DagRobotsBundle.php new file mode 100644 index 0000000..5d15ba1 --- /dev/null +++ b/RobotsBundle/DagRobotsBundle.php @@ -0,0 +1,36 @@ + + */ +class DagRobotsBundle extends AbstractResourceBundle +{ + /** + * {@inheritdoc} + */ + public static function getSupportedDrivers() + { + return array(SyliusResourceBundle::DRIVER_DOCTRINE_ORM); + } + + /** + * {@inheritdoc} + */ + protected function getModelInterfaces() + { + return array('Dag\Component\Robots\Model\RuleInterface' => 'dag.robots.model.robot_rule.class'); + } + + /** + * {@inheritdoc} + */ + protected function getModelNamespace() + { + return 'Dag\Component\Robots\Model'; + } +} diff --git a/RobotsBundle/DependencyInjection/Configuration.php b/RobotsBundle/DependencyInjection/Configuration.php new file mode 100644 index 0000000..9fa3df9 --- /dev/null +++ b/RobotsBundle/DependencyInjection/Configuration.php @@ -0,0 +1,125 @@ +root('dag_robots'); + + $rootNode + ->children() + ->scalarNode('driver')->defaultValue(SyliusResourceBundle::DRIVER_DOCTRINE_ORM)->end() + ->scalarNode('rule_provider')->defaultValue('dag.robots.rule_provider.config')->end() + ->end() + ; + + $this->addValidationGroupsSection($rootNode); + $this->addClassesSection($rootNode); + $this->addRulesSection($rootNode); + + return $treeBuilder; + } + + /** + * Adds `validation_groups` section. + * + * @param ArrayNodeDefinition $node + */ + private function addValidationGroupsSection(ArrayNodeDefinition $node) + { + $node + ->children() + ->arrayNode('validation_groups') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('robot_rule') + ->prototype('scalar')->end() + ->defaultValue(array('sylius')) + ->end() + ->end() + ->end() + ->end() + ; + } + + /** + * Adds `classes` section. + * + * @param ArrayNodeDefinition $node + */ + private function addClassesSection(ArrayNodeDefinition $node) + { + $node + ->children() + ->arrayNode('classes') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('robot_rule') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('model')->defaultValue('Dag\Component\Robots\Model\Rule')->end() + ->scalarNode('controller')->defaultValue('Sylius\Bundle\ResourceBundle\Controller\ResourceController')->end() + ->scalarNode('repository')->defaultValue('Dag\Bundle\RobotsBundle\Doctrine\ORM\RuleRepository')->end() + ->arrayNode('form') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('default')->defaultValue('Dag\Bundle\RobotsBundle\Form\Type\RuleType')->end() + ->end() + ->end() + ->end() + ->end() + ->end() + ->end() + ->end() + ; + } + + /** + * Adds `rules` section. + * + * @param ArrayNodeDefinition $node + */ + private function addRulesSection(ArrayNodeDefinition $node) + { + $node + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('rules') + ->prototype('array') + ->children() + ->scalarNode('route') + ->isRequired() + ->cannotBeEmpty() + ->end() + ->arrayNode('tags') + ->prototype('enum')->values($this->getTags())->end() + ->isRequired() + ->cannotBeEmpty() + ->end() + ->arrayNode('hosts') + ->prototype('scalar')->end() + ->cannotBeEmpty() + ->end() + ->end() + ->end() + ->end() + ->end() + ; + } + + /** + * @return array + */ + private function getTags() + { + return array('noindex', 'nofollow', 'none', 'noarchive', 'nosnippet', 'noodp', 'notranslate', 'noimageindex'); + } +} diff --git a/RobotsBundle/DependencyInjection/DagRobotsExtension.php b/RobotsBundle/DependencyInjection/DagRobotsExtension.php new file mode 100644 index 0000000..a1e90a0 --- /dev/null +++ b/RobotsBundle/DependencyInjection/DagRobotsExtension.php @@ -0,0 +1,57 @@ +configure( + $config, + new Configuration(), + $container, + self::CONFIGURE_LOADER | self::CONFIGURE_DATABASE | self::CONFIGURE_PARAMETERS | self::CONFIGURE_VALIDATORS | self::CONFIGURE_FORMS + ); + + $rules = array(); + + foreach ($config['rules'] as $rule) { + foreach ($rule['hosts'] as $host) { + $rules[$rule['route']][$host] = $rule['tags']; + } + } + + $container->setParameter('dag.robots.rules', $rules); + $container->setAlias('dag.robots.rule_provider', $config['rule_provider']); + } + + /** + * {@inheritdoc} + */ + public function prepend(ContainerBuilder $container) + { + if (!$container->hasExtension('doctrine_cache')) { + throw new \RuntimeException('DoctrineCacheBundle must be registered!'); + } + + $container->prependExtensionConfig('doctrine_cache', array( + 'providers' => array( + 'dag_robots' => '%sylius.cache%', + ), + )); + } +} diff --git a/RobotsBundle/Doctrine/ORM/RuleRepository.php b/RobotsBundle/Doctrine/ORM/RuleRepository.php new file mode 100644 index 0000000..690be12 --- /dev/null +++ b/RobotsBundle/Doctrine/ORM/RuleRepository.php @@ -0,0 +1,13 @@ + + */ +class RuleRepository extends EntityRepository implements RuleRepositoryInterface +{ +} diff --git a/RobotsBundle/Doctrine/RuleInitializer.php b/RobotsBundle/Doctrine/RuleInitializer.php new file mode 100644 index 0000000..6345ec1 --- /dev/null +++ b/RobotsBundle/Doctrine/RuleInitializer.php @@ -0,0 +1,71 @@ + + */ +class RuleInitializer +{ + /** + * @var array + */ + private $rules; + + /** + * @var ObjectManager + */ + private $ruleManager; + + /** + * @var RuleRepositoryInterface + */ + private $ruleRepository; + + public function __construct(array $rules, ObjectManager $ruleManager, RuleRepositoryInterface $ruleRepository) + { + $this->rules = $rules; + $this->ruleManager = $ruleManager; + $this->ruleRepository = $ruleRepository; + } + + public function initialize(OutputInterface $output = null) + { + try { + $this->initializeRules($output); + } catch (NonUniqueResultException $exception) { + if ($output) { + $output->writeln('Rules already initialized'); + } + } + } + + protected function initializeRules(OutputInterface $output = null) + { + foreach ($this->rules as $host => $rules) { + if (null === $rule = $this->ruleRepository->findOneBy(array('route' => $data['route']))) { + $rule = $this->ruleRepository->createNew(); + $rule->setRoute($data['route']); + $rule->setHosts($data['hosts']); + $rule->setTags($data['tags']); + if ($output) { + $output->writeln(sprintf( + 'Adding rule "%s". (Tags: %s, Hosts: %s)', + $data['route'], + implode(', ', $data['tags']), + implode(', ', $data['hosts']) + )); + } + } + + $this->ruleManager->persist($rule); + } + + $this->ruleManager->flush(); + } +} diff --git a/RobotsBundle/EventListener/KernelResponseListener.php b/RobotsBundle/EventListener/KernelResponseListener.php new file mode 100644 index 0000000..83dfb14 --- /dev/null +++ b/RobotsBundle/EventListener/KernelResponseListener.php @@ -0,0 +1,50 @@ +requestChecker = $requestChecker; + $this->tagResolver = $tagResolver; + } + + /** + * Add resolved tags to X-Robots-Tag response header. + * + * @param FilterResponseEvent $event + */ + public function addRobotTags(FilterResponseEvent $event) + { + if (!$event->isMasterRequest()) { + return; + } + + // if (!$this->requestChecker->isCrawler($event->getRequest())) { + // return; + // } + + if ($tags = $this->tagResolver->resolve($event->getRequest())) { + $event->getResponse()->headers->set('X-Robots-Tag', $tags); + } + } +} diff --git a/RobotsBundle/Form/Type/RuleType.php b/RobotsBundle/Form/Type/RuleType.php new file mode 100644 index 0000000..4db4d57 --- /dev/null +++ b/RobotsBundle/Form/Type/RuleType.php @@ -0,0 +1,29 @@ +add('route') + ->add('tags', 'dag_robots_tag_choice') + ->add('hosts') + ; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'dag_robots_robot_rule'; + } +} diff --git a/RobotsBundle/Form/Type/TagChoiceType.php b/RobotsBundle/Form/Type/TagChoiceType.php new file mode 100644 index 0000000..b0a8492 --- /dev/null +++ b/RobotsBundle/Form/Type/TagChoiceType.php @@ -0,0 +1,43 @@ + + */ +class TagChoiceType extends AbstractType +{ + /** + * {@inheritdoc} + */ + public function setDefaultOptions(OptionsResolverInterface $resolver) + { + $tags = array('noindex', 'nofollow', 'none', 'noarchive', 'nosnippet', 'noodp', 'notranslate', 'noimageindex'); + + $resolver + ->setDefaults(array( + 'choices' => $tags, + 'multiple' => true, + )) + ; + } + + /** + * {@inheritdoc} + */ + public function getParent() + { + return 'choice'; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'dag_robots_tag_choice'; + } +} diff --git a/RobotsBundle/Resources/config/doctrine/model/Rule.orm.xml b/RobotsBundle/Resources/config/doctrine/model/Rule.orm.xml new file mode 100644 index 0000000..8667e5e --- /dev/null +++ b/RobotsBundle/Resources/config/doctrine/model/Rule.orm.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/RobotsBundle/Resources/config/services.xml b/RobotsBundle/Resources/config/services.xml new file mode 100644 index 0000000..79ae920 --- /dev/null +++ b/RobotsBundle/Resources/config/services.xml @@ -0,0 +1,60 @@ + + + + + + Dag\Component\Robots\Resolver\TagResolver + Dag\Component\Robots\Checker\RequestChecker + Dag\Bundle\RobotsBundle\EventListener\KernelResponseListener + Dag\Bundle\RobotsBundle\Doctrine\RuleInitializer + + Dag\Component\Robots\Provider\RepositoryRuleProvider + Dag\Component\Robots\Provider\CachedRuleProvider + Dag\Component\Robots\Provider\FallbackRuleProvider + Dag\Component\Robots\Provider\ArrayRuleProvider + + Dag\Bundle\RobotsBundle\Form\Type\TagChoiceType + + + + + + + + + + + + + + + + + %dag.robots.rules% + + + + + + + + + + + + + + + %dag.robots.rules% + + + + + + + + + + diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..2a190cd --- /dev/null +++ b/composer.json @@ -0,0 +1,35 @@ +{ + "name": "cdaguerre/robots-bundle", + "type": "symfony-bundle", + "description": "Add contextual robots.txt files and meta headers", + "homepage": "https://github.com/cdaguerre/robots-bundle", + "keywords": ["robot", "crawler", "meta", "bundle"], + "license": "MIT", + "authors": [ + { "name": "Christian Daguerre", "email": "christian@worldia.com"} + ], + "require": { + "php": ">=5.5.0" + }, + "require-dev": { + "phpspec/phpspec": "~2.0", + "symfony/framework-bundle": "^2.7" + }, + "suggest": { + "symfony/framework-bundle": "To use the Symfony bundle." + }, + "config": { + "bin-dir": "bin" + }, + "autoload": { + "psr-4": { + "Dag\\Component\\Robots\\": "Robots/", + "Dag\\Bundle\\RobotsBundle\\": "RobotsBundle/" + } + }, + "extra": { + "branch-alias": { + "dev-master": "0.1.x-dev" + } + } +} \ No newline at end of file diff --git a/phpspec.yml.dist b/phpspec.yml.dist new file mode 100644 index 0000000..0b08330 --- /dev/null +++ b/phpspec.yml.dist @@ -0,0 +1,12 @@ +suites: + Robots: + namespace: Dag\Component\Robots + psr4_prefix: Dag\Component\Robots + spec_path: Robots + src_path: Robots + + RobotsBundle: + namespace: Dag\Bundle\RobotsBundle + psr4_prefix: Dag\Bundle\RobotsBundle + spec_path: RobotsBundle + src_path: RobotsBundle \ No newline at end of file