Merge pull request #8 from vtsykun/cron-command-gc

Job and zip's archive GC

src/Packagist/WebBundle/Cron/Handler/CleanupDistDir.php

@@ -0,0 +1,82 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Packagist\WebBundle\Cron\Handler;
+
+use Packagist\WebBundle\Service\DistConfig;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Filesystem\Filesystem;
+
+/**
+ * Remove unused mirrored package's *.zip archives
+ */
+class CleanupDistDir
+{
+    const DEFAULT_PERIOD = 60; // 60 days
+
+    private $fs;
+    private $distConfig;
+    private $logger;
+
+    public function __construct(DistConfig $distConfig, LoggerInterface $logger)
+    {
+        $this->fs = new Filesystem();
+        $this->distConfig = $distConfig;
+        $this->logger = $logger;
+    }
+
+    public function __invoke(array $arguments = [])
+    {
+        $keepPeriod = $arguments['period'] ?? self::DEFAULT_PERIOD;
+        $expireDate = new \DateTime('now', new \DateTimeZone('UTC'));
+        $expireDate->modify(sprintf('-%d days', $keepPeriod));
+
+        if (null === $this->distConfig->getDistDir() || !$this->fs->exists($this->distConfig->getDistDir())) {
+            return [];
+        }
+
+        $root = realpath($this->distConfig->getDistDir());
+        $dir = new \RecursiveDirectoryIterator(
+            $root,
+            \FilesystemIterator::FOLLOW_SYMLINKS | \FilesystemIterator::SKIP_DOTS
+        );
+
+        $paths = [];
+        $filter = new \RecursiveCallbackFilterIterator(
+            $dir,
+            function (\SplFileInfo $current) use (&$paths, $expireDate) {
+                if (!$current->getRealPath()) {
+                    return false;
+                }
+                if ($current->isFile() && preg_match('/[a-f0-9]{40}\.zip$/', $current->getFilename())) {
+                    if ($current->getMTime() < $expireDate->getTimestamp()) {
+                        $paths[] = $current->getRealPath();
+                    }
+                    return false;
+                }
+                if (is_dir($current->getPathname()) && 0 !== strpos($current->getPathname(), '.')) {
+                    return true;
+                }
+
+                return false;
+            }
+        );
+
+        $iterator = new \RecursiveIteratorIterator($filter);
+        $iterator->rewind();
+        if ($paths) {
+            $this->logger->info(sprintf('Unused %s *.zip archives was found', count($paths)), ['paths' => $paths]);
+        }
+
+        foreach ($paths as $path) {
+            try {
+                $this->fs->remove($path);
+            } catch (\Exception $exception) {
+                $this->logger->warning(sprintf('Unable to delete the file "%s", cause %s', $path, $exception->getMessage()), ['path' => $path, 'e' => $exception]);
+            }
+        }
+
+        return $paths;
+    }
+}

src/Packagist/WebBundle/Cron/Handler/CleanupJobStorage.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Packagist\WebBundle\Cron\Handler;
+
+use Doctrine\Persistence\ManagerRegistry;
+use Packagist\WebBundle\Entity\Job;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Cron command to cleanup jobs storage
+ */
+class CleanupJobStorage
+{
+    private $registry;
+    private $logger;
+
+    public function __construct(ManagerRegistry $registry, LoggerInterface $logger)
+    {
+        $this->registry = $registry;
+        $this->logger = $logger;
+    }
+
+    public function __invoke()
+    {
+        $keepPeriod = $this->selectKeepPeriod($count);
+        $expireDate = new \DateTime('now', new \DateTimeZone('UTC'));
+        $expireDate->modify(sprintf('-%d days', $keepPeriod));
+
+        $rowCount = $this->registry->getRepository(Job::class)
+            ->createQueryBuilder('j')
+            ->delete()
+            ->where('j.createdAt < :createdAt')
+            ->setParameter('createdAt', $expireDate)
+            ->getQuery()
+            ->execute();
+
+        $this->logger->info(sprintf('Removed %s jobs from storage, since: %s, jobs count: %s', $rowCount, $expireDate->format('c'), $count));
+
+        return [
+            'since' => $expireDate->format('c'),
+            'rows' => $rowCount,
+            'count' => $count
+        ];
+    }
+
+    protected function selectKeepPeriod(&$count = null)
+    {
+        $count = $this->registry->getRepository(Job::class)
+            ->createQueryBuilder('j')
+            ->resetDQLPart('select')
+            ->select('COUNT(j.id)')
+            ->getQuery()
+            ->getSingleScalarResult();
+
+        switch ($count) {
+            case $count > 60000:
+                return 2;
+            case $count > 40000:
+                return 5;
+            case $count > 25000:
+                return 10;
+            case $count > 10000:
+                return 21;
+        }
+
+        return 60;
+    }
+}

src/Packagist/WebBundle/Model/ValidatingArrayLoader.php

@@ -3,8 +3,9 @@
 namespace Packagist\WebBundle\Model;
 
 use Composer\Spdx\SpdxLicenses;
+use Composer\Package\Loader\ValidatingArrayLoader as ComposerValidatingArrayLoader;
 
-class ValidatingArrayLoader extends \Composer\Package\Loader\ValidatingArrayLoader
+class ValidatingArrayLoader extends ComposerValidatingArrayLoader
 {
     /**
      * {@inheritdoc}

src/Packagist/WebBundle/Resources/config/services.yml

@@ -297,3 +297,17 @@ services:
             - { name: kernel.event_listener, event: packagePersist }
             - { name: kernel.event_listener, event: packageError }
             - { name: kernel.event_listener, event: packageRemove }
+
+    Packagist\WebBundle\Cron\Handler\CleanupDistDir:
+        arguments:
+            - '@packagist.dist_config'
+            - '@logger'
+        tags:
+            - { name: okvpn.cron, cron: '40 1 */2 * *' }
+
+    Packagist\WebBundle\Cron\Handler\CleanupJobStorage:
+        arguments:
+            - '@doctrine'
+            - '@logger'
+        tags:
+            - { name: okvpn.cron, cron: '49 0 * * *' }

src/Packagist/WebBundle/Service/DistConfig.php

@@ -24,9 +24,15 @@ public function __construct(RouterInterface $router, array $config)
     public function generateTargetDir(string $name)
     {
         $intermediatePath = \preg_replace('#[^a-z0-9-_/]#i', '-', $name);
-        $targetDir = \sprintf('%s/%s', $this->config['basedir'], $intermediatePath);
+        return \sprintf('%s/%s', $this->config['basedir'], $intermediatePath);
+    }
 
-        return $targetDir;
+    /**
+     * @return string|null
+     */
+    public function getDistDir(): ?string
+    {
+        return $this->config['basedir'] ?? null;
     }
 
     /**

src/Packagist/WebBundle/Service/DistManager.php

@@ -34,6 +34,10 @@ public function getDistPath(Version $version): ?string
 
         $path = $this->config->generateDistFileName($version->getName(), $dist['reference'], $version->getVersion());
         if ($this->fileSystem->exists($path)) {
+            try {
+                $this->fileSystem->touch($path);
+            } catch (IOException $exception) {}
+
             return $path;
         }