Merge pull request #2082 from acelaya-forks/feature/orphan-visits-counts

Track orphan visits counts

config/autoload/entity-manager.global.php

@@ -5,6 +5,7 @@
 use Doctrine\ORM\Events;
 use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
 use Shlinkio\Shlink\Core\Config\EnvVars;
+use Shlinkio\Shlink\Core\Visit\Listener\OrphanVisitsCountTracker;
 use Shlinkio\Shlink\Core\Visit\Listener\ShortUrlVisitsCountTracker;
 
 use function Shlinkio\Shlink\Core\ArrayUtils\contains;
@@ -63,8 +64,8 @@
                 'load_mappings_using_functional_style' => true,
                 'default_repository_classname' => EntitySpecificationRepository::class,
                 'listeners' => [
-                    Events::onFlush => [ShortUrlVisitsCountTracker::class],
-                    Events::postFlush => [ShortUrlVisitsCountTracker::class],
+                    Events::onFlush => [ShortUrlVisitsCountTracker::class, OrphanVisitsCountTracker::class],
+                    Events::postFlush => [ShortUrlVisitsCountTracker::class, OrphanVisitsCountTracker::class],
                 ],
             ],
             'connection' => $resolveConnection(),

module/Core/config/dependencies.config.php

@@ -77,6 +77,7 @@
                 Visit\Entity\Visit::class,
             ],
             Visit\Listener\ShortUrlVisitsCountTracker::class => InvokableFactory::class,
+            Visit\Listener\OrphanVisitsCountTracker::class => InvokableFactory::class,
 
             Util\DoctrineBatchHelper::class => ConfigAbstractFactory::class,
             Util\RedirectResponseHelper::class => ConfigAbstractFactory::class,

module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Visit.Entity.OrphanVisitsCount.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core;
+
+use Doctrine\DBAL\Types\Types;
+use Doctrine\ORM\Mapping\Builder\ClassMetadataBuilder;
+use Doctrine\ORM\Mapping\ClassMetadata;
+
+return static function (ClassMetadata $metadata, array $emConfig): void {
+    $builder = new ClassMetadataBuilder($metadata);
+
+    $builder->setTable(determineTableName('orphan_visits_counts', $emConfig))
+            ->setCustomRepositoryClass(Visit\Repository\OrphanVisitsCountRepository::class);
+
+    $builder->createField('id', Types::BIGINT)
+            ->columnName('id')
+            ->makePrimaryKey()
+            ->generatedValue('IDENTITY')
+            ->option('unsigned', true)
+            ->build();
+
+    $builder->createField('potentialBot', Types::BOOLEAN)
+            ->columnName('potential_bot')
+            ->option('default', false)
+            ->build();
+
+    $builder->createField('count', Types::BIGINT)
+            ->columnName('count')
+            ->option('unsigned', true)
+            ->option('default', 1)
+            ->build();
+
+    $builder->createField('slotId', Types::INTEGER)
+            ->columnName('slot_id')
+            ->option('unsigned', true)
+            ->build();
+
+    $builder->addUniqueConstraint(['potential_bot', 'slot_id'], 'UQ_slot');
+};

module/Core/migrations/Version20240331111103.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Platforms\MySQLPlatform;
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Create a new orphan_visits_counts that will work similarly to the short_url_visits_counts
+ */
+final class Version20240331111103 extends AbstractMigration
+{
+    public function up(Schema $schema): void
+    {
+        $this->skipIf($schema->hasTable('orphan_visits_counts'));
+
+        $table = $schema->createTable('orphan_visits_counts');
+        $table->addColumn('id', Types::BIGINT, [
+            'unsigned' => true,
+            'autoincrement' => true,
+            'notnull' => true,
+        ]);
+        $table->setPrimaryKey(['id']);
+
+        $table->addColumn('potential_bot', Types::BOOLEAN, ['default' => false]);
+
+        $table->addColumn('slot_id', Types::INTEGER, [
+            'unsigned' => true,
+            'notnull' => true,
+            'default' => 1,
+        ]);
+
+        $table->addColumn('count', Types::BIGINT, [
+            'unsigned' => true,
+            'notnull' => true,
+            'default' => 1,
+        ]);
+
+        $table->addUniqueIndex(['potential_bot', 'slot_id'], 'UQ_slot');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->skipIf(! $schema->hasTable('orphan_visits_counts'));
+        $schema->dropTable('orphan_visits_counts');
+    }
+
+    public function isTransactional(): bool
+    {
+        return ! ($this->connection->getDatabasePlatform() instanceof MySQLPlatform);
+    }
+}

module/Core/migrations/Version20240331111447.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20240331111447 extends AbstractMigration
+{
+    public function up(Schema $schema): void
+    {
+        $visitsQb = $this->connection->createQueryBuilder();
+        $visitsQb->select('COUNT(id)')
+                 ->from('visits')
+                 ->where($visitsQb->expr()->isNull('short_url_id'))
+                 ->andWhere($visitsQb->expr()->eq('potential_bot', ':potential_bot'));
+
+        $botsCount = $visitsQb->setParameter('potential_bot', '1')->executeQuery()->fetchOne();
+        $nonBotsCount = $visitsQb->setParameter('potential_bot', '0')->executeQuery()->fetchOne();
+
+        if ($botsCount > 0) {
+            $this->insertCount($botsCount, potentialBot: true);
+        }
+        if ($nonBotsCount > 0) {
+            $this->insertCount($nonBotsCount, potentialBot: false);
+        }
+    }
+
+    private function insertCount(string|int $count, bool $potentialBot): void
+    {
+        $this->connection->createQueryBuilder()
+            ->insert('orphan_visits_counts')
+            ->values([
+                'count' => ':count',
+                'potential_bot' => ':potential_bot',
+            ])
+            ->setParameters([
+                'count' => $count,
+                'potential_bot' => $potentialBot ? '1' : '0',
+            ])
+            ->executeStatement();
+    }
+}

module/Core/src/Visit/Entity/OrphanVisitsCount.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Entity;
+
+use Shlinkio\Shlink\Common\Entity\AbstractEntity;
+
+class OrphanVisitsCount extends AbstractEntity
+{
+    public function __construct(
+        public readonly bool $potentialBot = false,
+        public readonly int $slotId = 1,
+        public readonly string $count = '1',
+    ) {
+    }
+}

module/Core/src/Visit/Listener/OrphanVisitsCountTracker.php

@@ -0,0 +1,145 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Listener;
+
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Exception;
+use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
+use Doctrine\DBAL\Platforms\SQLitePlatform;
+use Doctrine\DBAL\Platforms\SQLServerPlatform;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\ORM\Event\OnFlushEventArgs;
+use Doctrine\ORM\Event\PostFlushEventArgs;
+use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+
+use function rand;
+
+final class OrphanVisitsCountTracker
+{
+    /** @var object[] */
+    private array $entitiesToBeCreated = [];
+
+    public function onFlush(OnFlushEventArgs $args): void
+    {
+        // Track entities that are going to be created during this flush operation
+        $this->entitiesToBeCreated = $args->getObjectManager()->getUnitOfWork()->getScheduledEntityInsertions();
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function postFlush(PostFlushEventArgs $args): void
+    {
+        $em = $args->getObjectManager();
+        $entitiesToBeCreated = $this->entitiesToBeCreated;
+
+        // Reset tracked entities until next flush operation
+        $this->entitiesToBeCreated = [];
+
+        foreach ($entitiesToBeCreated as $entity) {
+            $this->trackVisitCount($em, $entity);
+        }
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function trackVisitCount(EntityManagerInterface $em, object $entity): void
+    {
+        // This is not an orphan visit
+        if (! $entity instanceof Visit || ! $entity->isOrphan()) {
+            return;
+        }
+        $visit = $entity;
+
+        $isBot = $visit->potentialBot;
+        $conn = $em->getConnection();
+        $platformClass = $conn->getDatabasePlatform();
+
+        match ($platformClass::class) {
+            PostgreSQLPlatform::class => $this->incrementForPostgres($conn, $isBot),
+            SQLitePlatform::class, SQLServerPlatform::class => $this->incrementForOthers($conn, $isBot),
+            default => $this->incrementForMySQL($conn, $isBot),
+        };
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForMySQL(Connection $conn, bool $potentialBot): void
+    {
+        $this->incrementWithPreparedStatement($conn, $potentialBot, <<<QUERY
+            INSERT INTO orphan_visits_counts (potential_bot, slot_id, count)
+            VALUES (:potential_bot, RAND() * 100, 1)
+            ON DUPLICATE KEY UPDATE count = count + 1;
+            QUERY);
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForPostgres(Connection $conn, bool $potentialBot): void
+    {
+        $this->incrementWithPreparedStatement($conn, $potentialBot, <<<QUERY
+            INSERT INTO orphan_visits_counts (potential_bot, slot_id, count)
+            VALUES (:potential_bot, random() * 100, 1)
+            ON CONFLICT (potential_bot, slot_id) DO UPDATE
+              SET count = orphan_visits_counts.count + 1;
+            QUERY);
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementWithPreparedStatement(Connection $conn, bool $potentialBot, string $query): void
+    {
+        $statement = $conn->prepare($query);
+        $statement->bindValue('potential_bot', $potentialBot ? 1 : 0);
+        $statement->executeStatement();
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function incrementForOthers(Connection $conn, bool $potentialBot): void
+    {
+        $slotId = rand(1, 100);
+
+        // For engines without a specific UPSERT syntax, do a regular locked select followed by an insert or update
+        $qb = $conn->createQueryBuilder();
+        $qb->select('id')
+           ->from('orphan_visits_counts')
+           ->where($qb->expr()->and(
+               $qb->expr()->eq('potential_bot', ':potential_bot'),
+               $qb->expr()->eq('slot_id', ':slot_id'),
+           ))
+           ->setParameter('potential_bot', $potentialBot ? '1' : '0')
+           ->setParameter('slot_id', $slotId)
+           ->setMaxResults(1);
+
+        if ($conn->getDatabasePlatform()::class === SQLServerPlatform::class) {
+            $qb->forUpdate();
+        }
+
+        $visitsCountId = $qb->executeQuery()->fetchOne();
+
+        $writeQb = ! $visitsCountId
+            ? $conn->createQueryBuilder()
+                ->insert('orphan_visits_counts')
+                ->values([
+                    'potential_bot' => ':potential_bot',
+                    'slot_id' => ':slot_id',
+                ])
+                ->setParameter('potential_bot', $potentialBot ? '1' : '0')
+                ->setParameter('slot_id', $slotId)
+            : $conn->createQueryBuilder()
+                ->update('orphan_visits_counts')
+                ->set('count', 'count + 1')
+                ->where($qb->expr()->eq('id', ':visits_count_id'))
+                ->setParameter('visits_count_id', $visitsCountId);
+
+        $writeQb->executeStatement();
+    }
+}

module/Core/src/Visit/Repository/OrphanVisitsCountRepository.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Repository;
+
+use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
+use Shlinkio\Shlink\Core\Visit\Entity\OrphanVisitsCount;
+use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering;
+use Shlinkio\Shlink\Rest\ApiKey\Role;
+
+class OrphanVisitsCountRepository extends EntitySpecificationRepository implements OrphanVisitsCountRepositoryInterface
+{
+    public function countOrphanVisits(VisitsCountFiltering $filtering): int
+    {
+        if ($filtering->apiKey?->hasRole(Role::NO_ORPHAN_VISITS)) {
+            return 0;
+        }
+
+        $qb = $this->getEntityManager()->createQueryBuilder();
+        $qb->select('COALESCE(SUM(vc.count), 0)')
+           ->from(OrphanVisitsCount::class, 'vc');
+
+        if ($filtering->excludeBots) {
+            $qb->andWhere($qb->expr()->eq('vc.potentialBot', ':potentialBot'))
+               ->setParameter('potentialBot', false);
+        }
+
+        return (int) $qb->getQuery()->getSingleScalarResult();
+    }
+}

module/Core/src/Visit/Repository/OrphanVisitsCountRepositoryInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Visit\Repository;
+
+use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering;
+
+interface OrphanVisitsCountRepositoryInterface
+{
+    public function countOrphanVisits(VisitsCountFiltering $filtering): int;
+}