Track orphan visits counts

config/autoload/entity-manager.global.php

@@ -5,6 +5,7 @@
 use Doctrine\ORM\Events;
 use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
 use Shlinkio\Shlink\Core\Config\EnvVars;
+use Shlinkio\Shlink\Core\Visit\Listener\OrphanVisitsCountTracker;
 use Shlinkio\Shlink\Core\Visit\Listener\ShortUrlVisitsCountTracker;
 
 use function Shlinkio\Shlink\Core\ArrayUtils\contains;
@@ -63,8 +64,8 @@
                 'load_mappings_using_functional_style' => true,
                 'default_repository_classname' => EntitySpecificationRepository::class,
                 'listeners' => [
-                    Events::onFlush => [ShortUrlVisitsCountTracker::class],
-                    Events::postFlush => [ShortUrlVisitsCountTracker::class],
+                    Events::onFlush => [ShortUrlVisitsCountTracker::class, OrphanVisitsCountTracker::class],
+                    Events::postFlush => [ShortUrlVisitsCountTracker::class, OrphanVisitsCountTracker::class],
                 ],
             ],
             'connection' => $resolveConnection(),

module/Core/config/dependencies.config.php

@@ -77,6 +77,7 @@
                 Visit\Entity\Visit::class,
             ],
             Visit\Listener\ShortUrlVisitsCountTracker::class => InvokableFactory::class,
+            Visit\Listener\OrphanVisitsCountTracker::class => InvokableFactory::class,
 
             Util\DoctrineBatchHelper::class => ConfigAbstractFactory::class,
             Util\RedirectResponseHelper::class => ConfigAbstractFactory::class,

module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Visit.Entity.OrphanVisitsCount.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core;
+
+use Doctrine\DBAL\Types\Types;
+use Doctrine\ORM\Mapping\Builder\ClassMetadataBuilder;
+use Doctrine\ORM\Mapping\ClassMetadata;
+
+return static function (ClassMetadata $metadata, array $emConfig): void {
+    $builder = new ClassMetadataBuilder($metadata);
+
+    $builder->setTable(determineTableName('orphan_visits_counts', $emConfig))
+            ->setCustomRepositoryClass(Visit\Repository\OrphanVisitsCountRepository::class);
+
+    $builder->createField('id', Types::BIGINT)
+            ->columnName('id')
+            ->makePrimaryKey()
+            ->generatedValue('IDENTITY')
+            ->option('unsigned', true)
+            ->build();
+
+    $builder->createField('potentialBot', Types::BOOLEAN)
+            ->columnName('potential_bot')
+            ->option('default', false)
+            ->build();
+
+    $builder->createField('count', Types::BIGINT)
+            ->columnName('count')
+            ->option('unsigned', true)
+            ->option('default', 1)
+            ->build();
+
+    $builder->createField('slotId', Types::INTEGER)
+            ->columnName('slot_id')
+            ->option('unsigned', true)
+            ->build();
+
+    $builder->addUniqueConstraint(['potential_bot', 'slot_id'], 'UQ_slot');
+};

module/Core/src/Visit/Entity/OrphanVisitsCount.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Entity;
+
+use Shlinkio\Shlink\Common\Entity\AbstractEntity;
+
+class OrphanVisitsCount extends AbstractEntity
+{
+    public function __construct(
+        public readonly bool $potentialBot = false,
+        public readonly int $slotId = 1,
+        public readonly string $count = '1',
+    ) {
+    }
+}

module/Core/src/Visit/Listener/OrphanVisitsCountTracker.php

@@ -0,0 +1,145 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Listener;
+
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Exception;
+use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
+use Doctrine\DBAL\Platforms\SQLitePlatform;
+use Doctrine\DBAL\Platforms\SQLServerPlatform;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\ORM\Event\OnFlushEventArgs;
+use Doctrine\ORM\Event\PostFlushEventArgs;
+use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+
+use function rand;
+
+final class OrphanVisitsCountTracker
+{
+    /** @var object[] */
+    private array $entitiesToBeCreated = [];
+
+    public function onFlush(OnFlushEventArgs $args): void
+    {
+        // Track entities that are going to be created during this flush operation
+        $this->entitiesToBeCreated = $args->getObjectManager()->getUnitOfWork()->getScheduledEntityInsertions();
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function postFlush(PostFlushEventArgs $args): void
+    {
+        $em = $args->getObjectManager();
+        $entitiesToBeCreated = $this->entitiesToBeCreated;
+
+        // Reset tracked entities until next flush operation
+        $this->entitiesToBeCreated = [];
+
+        foreach ($entitiesToBeCreated as $entity) {
+            $this->trackVisitCount($em, $entity);
+        }
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function trackVisitCount(EntityManagerInterface $em, object $entity): void
+    {
+        // This is not an orphan visit
+        if (! $entity instanceof Visit || ! $entity->isOrphan()) {
+            return;
+        }
+        $visit = $entity;
+
+        $isBot = $visit->potentialBot;
+        $conn = $em->getConnection();
+        $platformClass = $conn->getDatabasePlatform();
+
+        match ($platformClass::class) {
+            PostgreSQLPlatform::class => $this->incrementForPostgres($conn, $isBot),
+            SQLitePlatform::class, SQLServerPlatform::class => $this->incrementForOthers($conn, $isBot),
+            default => $this->incrementForMySQL($conn, $isBot),
+        };
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForMySQL(Connection $conn, bool $potentialBot): void
+    {
+        $this->incrementWithPreparedStatement($conn, $potentialBot, <<<QUERY
+            INSERT INTO orphan_visits_counts (potential_bot, slot_id, count)
+            VALUES (:potential_bot, RAND() * 100, 1)
+            ON DUPLICATE KEY UPDATE count = count + 1;
+            QUERY);
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForPostgres(Connection $conn, bool $potentialBot): void
+    {
+        $this->incrementWithPreparedStatement($conn, $potentialBot, <<<QUERY
+            INSERT INTO orphan_visits_counts (potential_bot, slot_id, count)
+            VALUES (:potential_bot, random() * 100, 1)
+            ON CONFLICT (potential_bot, slot_id) DO UPDATE
+              SET count = orphan_visits_counts.count + 1;
+            QUERY);
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementWithPreparedStatement(Connection $conn, bool $potentialBot, string $query): void
+    {
+        $statement = $conn->prepare($query);
+        $statement->bindValue('potential_bot', $potentialBot ? 1 : 0);
+        $statement->executeStatement();
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForOthers(Connection $conn, bool $potentialBot): void
+    {
+        $slotId = rand(1, 100);
+
+        // For engines without a specific UPSERT syntax, do a regular locked select followed by an insert or update
+        $qb = $conn->createQueryBuilder();
+        $qb->select('id')
+           ->from('orphan_visits_counts')
+           ->where($qb->expr()->and(
+               $qb->expr()->eq('potential_bot', ':potential_bot'),
+               $qb->expr()->eq('slot_id', ':slot_id'),
+           ))
+           ->setParameter('potential_bot', $potentialBot ? '1' : '0')
+           ->setParameter('slot_id', $slotId)
+           ->setMaxResults(1);
+
+        if ($conn->getDatabasePlatform()::class === SQLServerPlatform::class) {
+            $qb->forUpdate();
+        }
+
+        $visitsCountId = $qb->executeQuery()->fetchOne();
+
+        $writeQb = ! $visitsCountId
+            ? $conn->createQueryBuilder()
+                ->insert('orphan_visits_counts')
+                ->values([
+                    'potential_bot' => ':potential_bot',
+                    'slot_id' => ':slot_id',
+                ])
+                ->setParameter('potential_bot', $potentialBot ? '1' : '0')
+                ->setParameter('slot_id', $slotId)
+            : $conn->createQueryBuilder()
+                ->update('orphan_visits_counts')
+                ->set('count', 'count + 1')
+                ->where($qb->expr()->eq('id', ':visits_count_id'))
+                ->setParameter('visits_count_id', $visitsCountId);
+
+        $writeQb->executeStatement();
+    }
+}

module/Core/src/Visit/Repository/OrphanVisitsCountRepository.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Repository;
+
+use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
+use Shlinkio\Shlink\Core\Visit\Entity\OrphanVisitsCount;
+use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering;
+use Shlinkio\Shlink\Rest\ApiKey\Role;
+
+class OrphanVisitsCountRepository extends EntitySpecificationRepository implements OrphanVisitsCountRepositoryInterface
+{
+    public function countOrphanVisits(VisitsCountFiltering $filtering): int
+    {
+        if ($filtering->apiKey?->hasRole(Role::NO_ORPHAN_VISITS)) {
+            return 0;
+        }
+
+        $qb = $this->getEntityManager()->createQueryBuilder();
+        $qb->select('COALESCE(SUM(vc.count), 0)')
+           ->from(OrphanVisitsCount::class, 'vc');
+
+        if ($filtering->excludeBots) {
+            $qb->andWhere($qb->expr()->eq('vc.potentialBot', ':potentialBot'))
+               ->setParameter('potentialBot', false);
+        }
+
+        return (int) $qb->getQuery()->getSingleScalarResult();
+    }
+}

module/Core/src/Visit/Repository/OrphanVisitsCountRepositoryInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Repository;
+
+use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering;
+
+interface OrphanVisitsCountRepositoryInterface
+{
+    public function countOrphanVisits(VisitsCountFiltering $filtering): int;
+}

module/Core/src/Visit/VisitsStatsHelper.php

@@ -17,6 +17,7 @@
 use Shlinkio\Shlink\Core\ShortUrl\Repository\ShortUrlRepositoryInterface;
 use Shlinkio\Shlink\Core\Tag\Entity\Tag;
 use Shlinkio\Shlink\Core\Tag\Repository\TagRepository;
+use Shlinkio\Shlink\Core\Visit\Entity\OrphanVisitsCount;
 use Shlinkio\Shlink\Core\Visit\Entity\ShortUrlVisitsCount;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams;
@@ -29,8 +30,8 @@
 use Shlinkio\Shlink\Core\Visit\Paginator\Adapter\TagVisitsPaginatorAdapter;
 use Shlinkio\Shlink\Core\Visit\Persistence\OrphanVisitsCountFiltering;
 use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering;
+use Shlinkio\Shlink\Core\Visit\Repository\OrphanVisitsCountRepository;
 use Shlinkio\Shlink\Core\Visit\Repository\ShortUrlVisitsCountRepository;
-use Shlinkio\Shlink\Core\Visit\Repository\VisitRepository;
 use Shlinkio\Shlink\Core\Visit\Repository\VisitRepositoryInterface;
 use Shlinkio\Shlink\Rest\Entity\ApiKey;
 
@@ -42,18 +43,20 @@ public function __construct(private EntityManagerInterface $em)
 
     public function getVisitsStats(?ApiKey $apiKey = null): VisitsStats
     {
-        /** @var VisitRepository $visitsRepo */
-        $visitsRepo = $this->em->getRepository(Visit::class);
+        /** @var OrphanVisitsCountRepository $orphanVisitsCountRepo */
+        $orphanVisitsCountRepo = $this->em->getRepository(OrphanVisitsCount::class);
         /** @var ShortUrlVisitsCountRepository $visitsCountRepo */
         $visitsCountRepo = $this->em->getRepository(ShortUrlVisitsCount::class);
 
         return new VisitsStats(
             nonOrphanVisitsTotal: $visitsCountRepo->countNonOrphanVisits(new VisitsCountFiltering(apiKey: $apiKey)),
-            orphanVisitsTotal: $visitsRepo->countOrphanVisits(new OrphanVisitsCountFiltering(apiKey: $apiKey)),
+            orphanVisitsTotal: $orphanVisitsCountRepo->countOrphanVisits(
+                new OrphanVisitsCountFiltering(apiKey: $apiKey),
+            ),
             nonOrphanVisitsNonBots: $visitsCountRepo->countNonOrphanVisits(
                 new VisitsCountFiltering(excludeBots: true, apiKey: $apiKey),
             ),
-            orphanVisitsNonBots: $visitsRepo->countOrphanVisits(
+            orphanVisitsNonBots: $orphanVisitsCountRepo->countOrphanVisits(
                 new OrphanVisitsCountFiltering(excludeBots: true, apiKey: $apiKey),
             ),
         );

module/Core/test-db/Visit/Listener/OrphanVisitsCountTrackerTest.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioDbTest\Shlink\Core\Visit\Listener;
+
+use Doctrine\ORM\EntityRepository;
+use PHPUnit\Framework\Attributes\Test;
+use Shlinkio\Shlink\Core\Visit\Entity\OrphanVisitsCount;
+use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+use Shlinkio\Shlink\Core\Visit\Model\Visitor;
+use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase;
+
+use function array_filter;
+use function array_values;
+
+class OrphanVisitsCountTrackerTest extends DatabaseTestCase
+{
+    private EntityRepository $repo;
+
+    protected function setUp(): void
+    {
+        $this->repo = $this->getEntityManager()->getRepository(OrphanVisitsCount::class);
+    }
+
+    #[Test]
+    public function createsNewEntriesWhenNoneExist(): void
+    {
+        $visit = Visit::forBasePath(Visitor::emptyInstance());
+        $this->getEntityManager()->persist($visit);
+        $this->getEntityManager()->flush();
+
+        /** @var OrphanVisitsCount[] $result */
+        $result = $this->repo->findAll();
+
+        self::assertCount(1, $result);
+        self::assertEquals('1', $result[0]->count);
+        self::assertGreaterThanOrEqual(0, $result[0]->slotId);
+        self::assertLessThanOrEqual(100, $result[0]->slotId);
+    }
+
+    #[Test]
+    public function editsExistingEntriesWhenAlreadyExist(): void
+    {
+        for ($i = 0; $i <= 100; $i++) {
+            $this->getEntityManager()->persist(new OrphanVisitsCount(slotId: $i));
+        }
+        $this->getEntityManager()->flush();
+
+        $visit = Visit::forRegularNotFound(Visitor::emptyInstance());
+        $this->getEntityManager()->persist($visit);
+        $this->getEntityManager()->flush();
+
+        // Clear entity manager to force it to get fresh data from the database
+        // This is needed because the tracker inserts natively, bypassing the entity manager
+        $this->getEntityManager()->clear();
+
+        /** @var OrphanVisitsCount[] $result */
+        $result = $this->repo->findAll();
+        $itemsWithCountBiggerThanOnce = array_values(array_filter(
+            $result,
+            static fn (OrphanVisitsCount $item) => ((int) $item->count) > 1,
+        ));
+
+        self::assertCount(101, $result);
+        self::assertCount(1, $itemsWithCountBiggerThanOnce);
+        self::assertEquals('2', $itemsWithCountBiggerThanOnce[0]->count);
+    }
+}