From 2134a58ec6b9bebc9e08a65677089a486e2081b9 Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:54:58 +0300 Subject: [PATCH 1/9] UHF-10631: Add HelfiRecommendations entity --- ...hdbt_subtheme_aipoweredrecommendations.yml | 0 .../config/schema/helfi_annif.schema.yml | 12 ++++ .../custom/helfi_annif/helfi_annif.info.yml | 2 +- .../custom/helfi_annif/helfi_annif.install | 10 +++ .../custom/helfi_annif/helfi_annif.module | 9 ++- .../src/Entity/SuggestedTopics.php | 71 +++++++++++++++++++ .../FieldType/ScoredEntityReferenceItem.php | 70 ++++++++++++++++++ .../Plugin/QueueWorker/KeywordQueueWorker.php | 3 +- .../src/SuggestedTopicsInterface.php | 13 ++++ 9 files changed, 185 insertions(+), 5 deletions(-) rename public/modules/custom/helfi_annif/config/{install/public/modules/custom/helfi_annif/config => }/optional/block.block.hdbt_subtheme_aipoweredrecommendations.yml (100%) create mode 100644 public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml create mode 100644 public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php create mode 100644 public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php create mode 100644 public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php diff --git a/public/modules/custom/helfi_annif/config/install/public/modules/custom/helfi_annif/config/optional/block.block.hdbt_subtheme_aipoweredrecommendations.yml b/public/modules/custom/helfi_annif/config/optional/block.block.hdbt_subtheme_aipoweredrecommendations.yml similarity index 100% rename from public/modules/custom/helfi_annif/config/install/public/modules/custom/helfi_annif/config/optional/block.block.hdbt_subtheme_aipoweredrecommendations.yml rename to public/modules/custom/helfi_annif/config/optional/block.block.hdbt_subtheme_aipoweredrecommendations.yml diff --git a/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml b/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml new file mode 100644 index 000000000..134c5fb98 --- /dev/null +++ b/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml @@ -0,0 +1,12 @@ +field.value.helfi_annif_suggestion: + type: mapping + label: Default value + mapping: + target_id: + type: string + label: 'Value' + target_uuid: + type: uuid + score: + type: label + label: Score diff --git a/public/modules/custom/helfi_annif/helfi_annif.info.yml b/public/modules/custom/helfi_annif/helfi_annif.info.yml index 2f167d9e1..a9357aac1 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.info.yml +++ b/public/modules/custom/helfi_annif/helfi_annif.info.yml @@ -7,9 +7,9 @@ dependencies: - drupal:text - drupal:language - helfi_api_base:helfi_api_base - - helfi_etusivu:helfi_etusivu - readonly_field_widget:readonly_field_widget - drupal:field + - helfi_etusivu:helfi_etusivu - helfi_node_news_item:helfi_node_news_item - helfi_node_news_article:helfi_node_news_article 'interface translation project': helfi_annif diff --git a/public/modules/custom/helfi_annif/helfi_annif.install b/public/modules/custom/helfi_annif/helfi_annif.install index e1db7267c..0beb83b6d 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.install +++ b/public/modules/custom/helfi_annif/helfi_annif.install @@ -50,3 +50,13 @@ function helfi_annif_update_9001(): void { ); } } + +/** + * Creates the database table for the HelfiRecommendations entity. + */ +function helfi_annif_update_9002(): void { + $definition = \Drupal::entityTypeManager()->getDefinition('suggested_topics'); + + \Drupal::entityDefinitionUpdateManager() + ->installEntityType($definition); +} diff --git a/public/modules/custom/helfi_annif/helfi_annif.module b/public/modules/custom/helfi_annif/helfi_annif.module index 8571ca590..b7fb2855c 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.module +++ b/public/modules/custom/helfi_annif/helfi_annif.module @@ -34,7 +34,7 @@ function helfi_annif_theme() : array { /** * Implements hook_themes_installed(). */ -function helfi_annif_themes_installed($theme_list) { +function helfi_annif_themes_installed($theme_list): void { /** @var Drupal\helfi_platform_config\Helper\BlockInstaller $block_installer */ $block_installer = Drupal::service('helfi_platform_config.helper.block_installer'); @@ -74,7 +74,7 @@ function helfi_annif_themes_installed($theme_list) { * @return array[] * The block configurations. */ -function helfi_annif_get_block_configurations(string $theme) { +function helfi_annif_get_block_configurations(string $theme): array { return [ 'hdbt_subtheme_hdbt_subtheme_aipoweredrecommendations' => [ 'block' => [ @@ -172,7 +172,10 @@ function helfi_annif_entity_bundle_field_info_alter(&$fields, EntityTypeInterfac */ function helfi_annif_entity_field_storage_info(EntityTypeInterface $entity_type): array { if ($entity_type->id() === 'node') { - return array_merge(helfi_annif_bundle_fields($entity_type->id(), 'news_item'), helfi_annif_bundle_fields($entity_type->id(), 'news_article')); + return array_merge( + helfi_annif_bundle_fields($entity_type->id(), 'news_item'), + helfi_annif_bundle_fields($entity_type->id(), 'news_article') + ); } return []; } diff --git a/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php new file mode 100644 index 000000000..89229db8d --- /dev/null +++ b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php @@ -0,0 +1,71 @@ +setLabel(new TranslatableMarkup('Status')) + ->setDefaultValue(FALSE) + ->setSetting('on_label', 'Enabled') + ->setDisplayOptions('view', [ + 'type' => 'boolean', + 'label' => 'above', + 'weight' => 0, + 'settings' => [ + 'format' => 'enabled-disabled', + ], + ]) + ->setDisplayConfigurable('view', TRUE); + + $fields['keywords'] = BaseFieldDefinition::create('scored_entity_reference') + ->setLabel(new TranslatableMarkup('Keywords')) + ->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED) + ->setSetting('target_type', 'taxonomy_term') + ->setDisplayOptions('view', [ + 'label' => 'above', + 'type' => 'entity_reference_label', + 'weight' => 15, + ]) + ->setDisplayConfigurable('view', TRUE); + + return $fields; + } + +} diff --git a/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php new file mode 100644 index 000000000..fbdecbad6 --- /dev/null +++ b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php @@ -0,0 +1,70 @@ +setLabel(new TranslatableMarkup('Score')) + ->setRequired(TRUE); + + return $properties; + } + + /** + * {@inheritdoc} + */ + public static function schema(FieldStorageDefinitionInterface $field_definition): array { + $schema = parent::schema($field_definition); + + // Score is a decimal number between 0 and 1. + $schema['columns']['score'] = [ + 'type' => 'float', + ]; + + return $schema; + } + + /** + * {@inheritdoc} + */ + public static function generateSampleValue(FieldDefinitionInterface $field_definition): array { + $values = parent::generateSampleValue($field_definition); + + $values['score'] = mt_rand() / mt_getrandmax(); + + return $values; + } + +} diff --git a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php index 0f722e710..ec7b1a7ed 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php +++ b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php @@ -8,6 +8,7 @@ use Drupal\Core\Entity\TranslatableInterface; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Queue\QueueWorkerBase; +use Drupal\Core\Utility\Error; use Drupal\helfi_annif\Client\KeywordClientException; use Drupal\helfi_annif\KeywordManager; use Psr\Log\LoggerInterface; @@ -94,7 +95,7 @@ public function processItem(mixed $data) : void { $this->keywordManager->processEntity($entity, overwriteExisting: $overwrite); } catch (KeywordClientException $exception) { - $this->logger->error($exception->getMessage()); + Error::logException($this->logger, $exception); } } diff --git a/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php b/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php new file mode 100644 index 000000000..4288a4165 --- /dev/null +++ b/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php @@ -0,0 +1,13 @@ + Date: Tue, 24 Sep 2024 08:55:04 +0300 Subject: [PATCH 2/9] UHF-10631: Rename KeywordClient to ApiClient --- .../custom/helfi_annif/helfi_annif.services.yml | 2 +- .../Client/{KeywordClient.php => ApiClient.php} | 10 +++++----- ...lientException.php => ApiClientException.php} | 2 +- .../src/Drush/Commands/AnnifCommands.php | 4 ++-- .../custom/helfi_annif/src/KeywordManager.php | 13 +++++++------ .../Plugin/QueueWorker/KeywordQueueWorker.php | 4 ++-- .../tests/src/Kernel/KeywordManagerTest.php | 4 ++-- .../{KeywordClientTest.php => ApiClientTest.php} | 16 ++++++++-------- .../tests/src/Unit/KeywordManagerTest.php | 4 ++-- 9 files changed, 30 insertions(+), 29 deletions(-) rename public/modules/custom/helfi_annif/src/Client/{KeywordClient.php => ApiClient.php} (94%) rename public/modules/custom/helfi_annif/src/Client/{KeywordClientException.php => ApiClientException.php} (71%) rename public/modules/custom/helfi_annif/tests/src/Unit/Client/{KeywordClientTest.php => ApiClientTest.php} (90%) diff --git a/public/modules/custom/helfi_annif/helfi_annif.services.yml b/public/modules/custom/helfi_annif/helfi_annif.services.yml index 1c6c410c2..07cdb1b83 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.services.yml +++ b/public/modules/custom/helfi_annif/helfi_annif.services.yml @@ -11,7 +11,7 @@ services: Drupal\helfi_annif\RecommendationManager: ~ - Drupal\helfi_annif\Client\KeywordClient: ~ + Drupal\helfi_annif\Client\ApiClient : ~ Drupal\helfi_annif\TextConverter\TextConverterManager: tags: diff --git a/public/modules/custom/helfi_annif/src/Client/KeywordClient.php b/public/modules/custom/helfi_annif/src/Client/ApiClient.php similarity index 94% rename from public/modules/custom/helfi_annif/src/Client/KeywordClient.php rename to public/modules/custom/helfi_annif/src/Client/ApiClient.php index 31fa23d77..1d4cf5d32 100644 --- a/public/modules/custom/helfi_annif/src/Client/KeywordClient.php +++ b/public/modules/custom/helfi_annif/src/Client/ApiClient.php @@ -13,7 +13,7 @@ /** * The keyword generator. */ -final class KeywordClient { +final class ApiClient { /** * Maximum batch size. @@ -62,7 +62,7 @@ private function getDefaultOptions() : array { * @return \Drupal\helfi_annif\Client\Keyword[]|null * Keywords or NULL if unsupported entity. * - * @throws KeywordClientException + * @throws \Drupal\helfi_annif\Client\ApiClientException * If keyword generator returns an error. */ public function suggest(EntityInterface $entity) : ?array { @@ -93,7 +93,7 @@ public function suggest(EntityInterface $entity) : ?array { return $this->mapResults(Utils::jsonDecode($response->getBody()->getContents())); } catch (GuzzleException $e) { - throw new KeywordClientException($e->getMessage(), previous: $e); + throw new ApiClientException($e->getMessage(), previous: $e); } } @@ -107,7 +107,7 @@ public function suggest(EntityInterface $entity) : ?array { * Batch suggestion results keyed by input array keys. The * results array might not contain all input entities. * - * @throws KeywordClientException + * @throws \Drupal\helfi_annif\Client\ApiClientException * If keyword generator returns an error. */ public function suggestBatch(array $entities) : array { @@ -171,7 +171,7 @@ function ($carry, $item) { ); } catch (GuzzleException $e) { - throw new KeywordClientException($e->getMessage(), previous: $e); + throw new ApiClientException($e->getMessage(), previous: $e); } } diff --git a/public/modules/custom/helfi_annif/src/Client/KeywordClientException.php b/public/modules/custom/helfi_annif/src/Client/ApiClientException.php similarity index 71% rename from public/modules/custom/helfi_annif/src/Client/KeywordClientException.php rename to public/modules/custom/helfi_annif/src/Client/ApiClientException.php index e4096f7f5..44af97939 100644 --- a/public/modules/custom/helfi_annif/src/Client/KeywordClientException.php +++ b/public/modules/custom/helfi_annif/src/Client/ApiClientException.php @@ -7,5 +7,5 @@ /** * Exceptions for keyword generator errors. */ -class KeywordClientException extends \Exception { +class ApiClientException extends \Exception { } diff --git a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php index ce1729ff8..cb29124ce 100644 --- a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php +++ b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php @@ -13,7 +13,7 @@ use Drupal\Core\Entity\TranslatableInterface; use Drupal\Core\StringTranslation\StringTranslationTrait; use Drupal\Core\Utility\Error; -use Drupal\helfi_annif\Client\KeywordClient; +use Drupal\helfi_annif\Client\ApiClient; use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\TextConverter\TextConverterManager; use Drush\Attributes\Argument; @@ -79,7 +79,7 @@ public function process( string $bundle, array $options = [ 'overwrite' => FALSE, - 'batch-size' => KeywordClient::MAX_BATCH_SIZE, + 'batch-size' => ApiClient::MAX_BATCH_SIZE, ], ) : int { $definition = $this->entityTypeManager->getDefinition($entityType); diff --git a/public/modules/custom/helfi_annif/src/KeywordManager.php b/public/modules/custom/helfi_annif/src/KeywordManager.php index 8c1bc7e8f..447330f3e 100644 --- a/public/modules/custom/helfi_annif/src/KeywordManager.php +++ b/public/modules/custom/helfi_annif/src/KeywordManager.php @@ -9,8 +9,8 @@ use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Entity\TranslatableInterface; use Drupal\Core\Queue\QueueFactory; +use Drupal\helfi_annif\Client\ApiClient; use Drupal\helfi_annif\Client\Keyword; -use Drupal\helfi_annif\Client\KeywordClient; /** * The keyword manager. @@ -39,7 +39,7 @@ final class KeywordManager { * * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entityTypeManager * The entity type manager. - * @param \Drupal\helfi_annif\Client\KeywordClient $keywordGenerator + * @param \Drupal\helfi_annif\Client\ApiClient $keywordGenerator * The keyword generator. * @param \Drupal\Core\Queue\QueueFactory $queueFactory * The queue factory. @@ -49,7 +49,7 @@ final class KeywordManager { */ public function __construct( private readonly EntityTypeManagerInterface $entityTypeManager, - private readonly KeywordClient $keywordGenerator, + private readonly ApiClient $keywordGenerator, private readonly QueueFactory $queueFactory, ) { $this->termStorage = $this->entityTypeManager->getStorage('taxonomy_term'); @@ -115,7 +115,8 @@ public function queueEntity(RecommendableInterface $entity, bool $overwriteExist * @param bool $overwriteExisting * Overwrites existing keywords when set to TRUE. * - * @throws \Drupal\helfi_annif\Client\KeywordClientException + * @throws \Drupal\helfi_annif\Client\ApiClientException + * @throws \Drupal\Core\Entity\EntityStorageException */ public function processEntity(RecommendableInterface $entity, bool $overwriteExisting = FALSE) : void { if (!$entity->hasField(self::KEYWORD_FIELD)) { @@ -143,7 +144,7 @@ public function processEntity(RecommendableInterface $entity, bool $overwriteExi * @param bool $overwriteExisting * Overwrites existing keywords when set to TRUE. * - * @throws \Drupal\helfi_annif\Client\KeywordClientException + * @throws \Drupal\helfi_annif\Client\ApiClientException * @throws \Drupal\Core\Entity\EntityStorageException */ public function processEntities(array $entities, bool $overwriteExisting = FALSE) : void { @@ -201,7 +202,7 @@ private function prepareBatches(array $entities, bool $overwriteExisting) : \Gen } foreach ($buckets as $bucket) { - foreach (array_chunk($bucket, KeywordClient::MAX_BATCH_SIZE, preserve_keys: TRUE) as $batch) { + foreach (array_chunk($bucket, ApiClient::MAX_BATCH_SIZE, preserve_keys: TRUE) as $batch) { yield $batch; } } diff --git a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php index ec7b1a7ed..0f23e216b 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php +++ b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php @@ -9,7 +9,7 @@ use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Queue\QueueWorkerBase; use Drupal\Core\Utility\Error; -use Drupal\helfi_annif\Client\KeywordClientException; +use Drupal\helfi_annif\Client\ApiClientException; use Drupal\helfi_annif\KeywordManager; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -94,7 +94,7 @@ public function processItem(mixed $data) : void { try { $this->keywordManager->processEntity($entity, overwriteExisting: $overwrite); } - catch (KeywordClientException $exception) { + catch (ApiClientException $exception) { Error::logException($this->logger, $exception); } } diff --git a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php index a1870858e..5e2454f93 100644 --- a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php @@ -7,7 +7,7 @@ use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Queue\QueueFactory; use Drupal\Core\Queue\QueueInterface; -use Drupal\helfi_annif\Client\KeywordClient; +use Drupal\helfi_annif\Client\ApiClient; use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\TextConverter\TextConverterInterface; use Drupal\KernelTests\KernelTestBase; @@ -116,7 +116,7 @@ private function getSut( ): KeywordManager { $textConverterManager = $this->getTextConverterManager($textConverter); - $client = new KeywordClient( + $client = new ApiClient( $this->createMockHttpClient($responses), $textConverterManager, ); diff --git a/public/modules/custom/helfi_annif/tests/src/Unit/Client/KeywordClientTest.php b/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php similarity index 90% rename from public/modules/custom/helfi_annif/tests/src/Unit/Client/KeywordClientTest.php rename to public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php index 75bac35ed..e20374084 100644 --- a/public/modules/custom/helfi_annif/tests/src/Unit/Client/KeywordClientTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php @@ -4,9 +4,9 @@ namespace Drupal\Tests\helfi_annif\Unit\Client; +use Drupal\helfi_annif\Client\ApiClient; +use Drupal\helfi_annif\Client\ApiClientException; use Drupal\helfi_annif\Client\Keyword; -use Drupal\helfi_annif\Client\KeywordClient; -use Drupal\helfi_annif\Client\KeywordClientException; use Drupal\helfi_annif\TextConverter\TextConverterInterface; use Drupal\Tests\helfi_annif\Traits\AnnifApiTestTrait; use Drupal\Tests\UnitTestCase; @@ -78,7 +78,7 @@ public function testHttpError(): void { new RequestException('Bad request', new Request('GET', 'test')), ]); - $this->expectException(KeywordClientException::class); + $this->expectException(ApiClientException::class); $sut->suggest($entity); } @@ -92,7 +92,7 @@ public function testHttpErrorBatch(): void { new RequestException('Bad request', new Request('GET', 'test')), ]); - $this->expectException(KeywordClientException::class); + $this->expectException(ApiClientException::class); $sut->suggestBatch([$entity]); } @@ -150,7 +150,7 @@ public function testValidBatchRequest(): void { ]); $textConverterManager = $this->getTextConverterManager(); - $sut = new KeywordClient($httpClient, $textConverterManager); + $sut = new ApiClient($httpClient, $textConverterManager); $batch = $sut->suggestBatch($entities); @@ -175,7 +175,7 @@ public function testValidBatchRequest(): void { public function testMaxBatchSize() : void { $sut = $this->getSut([]); - $batch = array_fill(0, KeywordClient::MAX_BATCH_SIZE + 1, $this->mockEntity()); + $batch = array_fill(0, ApiClient::MAX_BATCH_SIZE + 1, $this->mockEntity()); $this->expectException(\InvalidArgumentException::class); $sut->suggestBatch($batch); @@ -184,11 +184,11 @@ public function testMaxBatchSize() : void { /** * Gets service under test. */ - private function getSut(array $responses, ?TextConverterInterface $textConverter = NULL): KeywordClient { + private function getSut(array $responses, ?TextConverterInterface $textConverter = NULL): ApiClient { $client = $this->createMockHttpClient($responses); $textConverterManager = $this->getTextConverterManager($textConverter); - return new KeywordClient($client, $textConverterManager); + return new ApiClient($client, $textConverterManager); } } diff --git a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php index 9451b5015..7d192eeae 100644 --- a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php @@ -8,7 +8,7 @@ use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Queue\QueueFactory; use Drupal\Core\Queue\QueueInterface; -use Drupal\helfi_annif\Client\KeywordClient; +use Drupal\helfi_annif\Client\ApiClient; use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\TextConverter\TextConverterInterface; use Drupal\Tests\helfi_annif\Traits\AnnifApiTestTrait; @@ -101,7 +101,7 @@ private function getSut( ): KeywordManager { $textConverterManager = $this->getTextConverterManager($textConverter); - $client = new KeywordClient( + $client = new ApiClient( $this->createMockHttpClient($responses), $textConverterManager, ); From 034bb97bc7ec4947e446e4d3fcfa1e9d8b05a8e0 Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:55:05 +0300 Subject: [PATCH 3/9] UHF-10631: Rename KeywordManager to TopicsManager --- .../modules/custom/helfi_annif/helfi_annif.module | 14 +++++++------- .../custom/helfi_annif/helfi_annif.services.yml | 2 +- .../src/Drush/Commands/AnnifCommands.php | 8 ++++---- .../src/Plugin/QueueWorker/KeywordQueueWorker.php | 10 +++++----- .../src/{KeywordManager.php => TopicsManager.php} | 4 ++-- .../tests/src/Kernel/KeywordManagerTest.php | 8 ++++---- .../tests/src/Traits/AnnifApiTestTrait.php | 6 +++--- .../tests/src/Unit/KeywordManagerTest.php | 12 ++++++------ 8 files changed, 32 insertions(+), 32 deletions(-) rename public/modules/custom/helfi_annif/src/{KeywordManager.php => TopicsManager.php} (99%) diff --git a/public/modules/custom/helfi_annif/helfi_annif.module b/public/modules/custom/helfi_annif/helfi_annif.module index b7fb2855c..d5eb328e8 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.module +++ b/public/modules/custom/helfi_annif/helfi_annif.module @@ -12,9 +12,9 @@ use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Field\FieldStorageDefinitionInterface; use Drupal\Core\StringTranslation\TranslatableMarkup; use Drupal\entity\BundleFieldDefinition; -use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\RecommendableInterface; use Drupal\helfi_annif\TextConverter\Document; +use Drupal\helfi_annif\TopicsManager; /** * Implements hook_theme(). @@ -132,9 +132,9 @@ function helfi_annif_get_block_configurations(string $theme): array { */ function helfi_annif_entity_insert(EntityInterface $entity) : void { if ($entity instanceof RecommendableInterface) { - /** @var \Drupal\helfi_annif\KeywordManager $keywordManager */ - $keywordManager = \Drupal::service(KeywordManager::class); - $keywordManager->queueEntity($entity, TRUE); + /** @var \Drupal\helfi_annif\TopicsManager $topicsManager */ + $topicsManager = \Drupal::service(TopicsManager::class); + $topicsManager->queueEntity($entity, TRUE); } } @@ -143,9 +143,9 @@ function helfi_annif_entity_insert(EntityInterface $entity) : void { */ function helfi_annif_entity_update(EntityInterface $entity) : void { if ($entity instanceof RecommendableInterface) { - /** @var \Drupal\helfi_annif\KeywordManager $keywordManager */ - $keywordManager = \Drupal::service(KeywordManager::class); - $keywordManager->queueEntity($entity, TRUE); + /** @var \Drupal\helfi_annif\TopicsManager $topicsManager */ + $topicsManager = \Drupal::service(TopicsManager::class); + $topicsManager->queueEntity($entity, TRUE); } } diff --git a/public/modules/custom/helfi_annif/helfi_annif.services.yml b/public/modules/custom/helfi_annif/helfi_annif.services.yml index 07cdb1b83..2aafb08cb 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.services.yml +++ b/public/modules/custom/helfi_annif/helfi_annif.services.yml @@ -7,7 +7,7 @@ services: parent: logger.channel_base arguments: ['helfi_annif'] - Drupal\helfi_annif\KeywordManager: ~ + Drupal\helfi_annif\TopicsManager: ~ Drupal\helfi_annif\RecommendationManager: ~ diff --git a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php index cb29124ce..a4592811e 100644 --- a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php +++ b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php @@ -14,7 +14,7 @@ use Drupal\Core\StringTranslation\StringTranslationTrait; use Drupal\Core\Utility\Error; use Drupal\helfi_annif\Client\ApiClient; -use Drupal\helfi_annif\KeywordManager; +use Drupal\helfi_annif\TopicsManager; use Drupal\helfi_annif\TextConverter\TextConverterManager; use Drush\Attributes\Argument; use Drush\Attributes\Command; @@ -41,14 +41,14 @@ final class AnnifCommands extends DrushCommands { * The entity type manager. * @param \Drupal\helfi_annif\TextConverter\TextConverterManager $textConverter * The text converter. - * @param \Drupal\helfi_annif\KeywordManager $keywordManager + * @param \Drupal\helfi_annif\TopicsManager $topicsManager * The keyword generator. */ public function __construct( private readonly Connection $connection, private readonly EntityTypeManagerInterface $entityTypeManager, private readonly TextConverterManager $textConverter, - private readonly KeywordManager $keywordManager, + private readonly TopicsManager $topicsManager, ) { parent::__construct(); } @@ -136,7 +136,7 @@ public function processBatch( ->getStorage($entityType) ->loadMultiple($slice); - $this->keywordManager->processEntities($entities, $overwrite); + $this->topicsManager->processEntities($entities, $overwrite); $context['sandbox']['from'] = $to; $context['message'] = $this->t("@total entities remaining", [ diff --git a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php index 0f23e216b..371564a03 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php +++ b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php @@ -10,7 +10,7 @@ use Drupal\Core\Queue\QueueWorkerBase; use Drupal\Core\Utility\Error; use Drupal\helfi_annif\Client\ApiClientException; -use Drupal\helfi_annif\KeywordManager; +use Drupal\helfi_annif\TopicsManager; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -28,9 +28,9 @@ final class KeywordQueueWorker extends QueueWorkerBase implements ContainerFacto /** * The keyword manager. * - * @var \Drupal\helfi_annif\KeywordManager + * @var \Drupal\helfi_annif\TopicsManager */ - private KeywordManager $keywordManager; + private TopicsManager $topicsManager; /** * The entity repository. @@ -55,7 +55,7 @@ public static function create(ContainerInterface $container, array $configuratio $plugin_id, $plugin_definition, ); - $instance->keywordManager = $container->get(KeywordManager::class); + $instance->topicsManager = $container->get(TopicsManager::class); $instance->entityTypeManager = $container->get(EntityTypeManagerInterface::class); $instance->logger = $container->get('logger.channel.helfi_annif'); @@ -92,7 +92,7 @@ public function processItem(mixed $data) : void { } try { - $this->keywordManager->processEntity($entity, overwriteExisting: $overwrite); + $this->topicsManager->processEntity($entity, overwriteExisting: $overwrite); } catch (ApiClientException $exception) { Error::logException($this->logger, $exception); diff --git a/public/modules/custom/helfi_annif/src/KeywordManager.php b/public/modules/custom/helfi_annif/src/TopicsManager.php similarity index 99% rename from public/modules/custom/helfi_annif/src/KeywordManager.php rename to public/modules/custom/helfi_annif/src/TopicsManager.php index 447330f3e..300e8e6e8 100644 --- a/public/modules/custom/helfi_annif/src/KeywordManager.php +++ b/public/modules/custom/helfi_annif/src/TopicsManager.php @@ -13,9 +13,9 @@ use Drupal\helfi_annif\Client\Keyword; /** - * The keyword manager. + * The topic manager. */ -final class KeywordManager { +final class TopicsManager { public const KEYWORD_FIELD = 'annif_keywords'; public const KEYWORD_VID = 'annif_keywords'; diff --git a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php index 5e2454f93..247fe00fd 100644 --- a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php @@ -8,8 +8,8 @@ use Drupal\Core\Queue\QueueFactory; use Drupal\Core\Queue\QueueInterface; use Drupal\helfi_annif\Client\ApiClient; -use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\TextConverter\TextConverterInterface; +use Drupal\helfi_annif\TopicsManager; use Drupal\KernelTests\KernelTestBase; use Drupal\language\Entity\ConfigurableLanguage; use Drupal\Tests\helfi_annif\Traits\AnnifApiTestTrait; @@ -17,7 +17,7 @@ use Prophecy\Argument; /** - * Tests KeywordManager. + * Tests TopicsManager. * * @group helfi_annif */ @@ -113,7 +113,7 @@ private function getSut( array $responses = [], ?TextConverterInterface $textConverter = NULL, ?QueueInterface $queue = NULL, - ): KeywordManager { + ): TopicsManager { $textConverterManager = $this->getTextConverterManager($textConverter); $client = new ApiClient( @@ -134,7 +134,7 @@ private function getSut( ->get(Argument::any()) ->willReturn($queue); - return new KeywordManager( + return new TopicsManager( $entityTypeManager, $client, $queueFactory->reveal(), diff --git a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php index afb31583e..9e174e35e 100644 --- a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php +++ b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php @@ -6,10 +6,10 @@ use Drupal\Core\Field\FieldItemListInterface; use Drupal\Core\Language\LanguageInterface; -use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\RecommendableInterface; use Drupal\helfi_annif\TextConverter\TextConverterInterface; use Drupal\helfi_annif\TextConverter\TextConverterManager; +use Drupal\helfi_annif\TopicsManager; use Drupal\Tests\helfi_api_base\Traits\ApiTestTrait; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; @@ -81,10 +81,10 @@ protected function mockEntity(string $langcode = 'fi', bool|NULL $hasKeywords = $field->isEmpty()->willReturn(!$hasKeywords); $entity - ->hasField(Argument::exact(KeywordManager::KEYWORD_FIELD)) + ->hasField(Argument::exact(TopicsManager::KEYWORD_FIELD)) ->willReturn($hasKeywords !== NULL); $entity - ->get(Argument::exact(KeywordManager::KEYWORD_FIELD)) + ->get(Argument::exact(TopicsManager::KEYWORD_FIELD)) ->willReturn($field->reveal()); if (is_bool($shouldSave)) { diff --git a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php index 7d192eeae..65c6a76a7 100644 --- a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php @@ -9,14 +9,14 @@ use Drupal\Core\Queue\QueueFactory; use Drupal\Core\Queue\QueueInterface; use Drupal\helfi_annif\Client\ApiClient; -use Drupal\helfi_annif\KeywordManager; use Drupal\helfi_annif\TextConverter\TextConverterInterface; +use Drupal\helfi_annif\TopicsManager; use Drupal\Tests\helfi_annif\Traits\AnnifApiTestTrait; use Drupal\Tests\UnitTestCase; use Prophecy\Argument; /** - * Tests KeywordManager. + * Tests TopicsManager. * * @group helfi_annif */ @@ -28,7 +28,7 @@ class KeywordManagerTest extends UnitTestCase { * Tests entities without keyword field. */ public function testUnsupportedEntity(): void { - // hasField(KeywordManager::KEYWORD_FIELD) for entity is FALSE. + // hasField(TopicsManager::KEYWORD_FIELD) for entity is FALSE. $entity = $this->mockEntity(hasKeywords: NULL, hasKeywordField: FALSE); $queue = $this->prophesize(QueueInterface::class); $queue @@ -67,7 +67,7 @@ public function testKeywordOverwriting(): void { * Tests entities with unsupported langcode. */ public function testUnsupportedLangcode(): void { - // hasField(KeywordManager::KEYWORD_FIELD) for entity is FALSE. + // hasField(TopicsManager::KEYWORD_FIELD) for entity is FALSE. $entity = $this->mockEntity(langcode: 'xzz', shouldSave: FALSE); $sut = $this->getSut(); @@ -98,7 +98,7 @@ private function getSut( ?TextConverterInterface $textConverter = NULL, ?EntityStorageInterface $termStorage = NULL, ?QueueInterface $queue = NULL, - ): KeywordManager { + ): TopicsManager { $textConverterManager = $this->getTextConverterManager($textConverter); $client = new ApiClient( @@ -128,7 +128,7 @@ private function getSut( ->get(Argument::any()) ->willReturn($queue); - return new KeywordManager( + return new TopicsManager( $entityTypeManager->reveal(), $client, $queueFactory->reveal(), From d97d60f3e0a2f83095a9651ab2103fff6b28671a Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:55:07 +0300 Subject: [PATCH 4/9] UHF-10631: Rename KeywordQueueWorker to QueueWorker --- .../QueueWorker/{KeywordQueueWorker.php => QueueWorker.php} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename public/modules/custom/helfi_annif/src/Plugin/QueueWorker/{KeywordQueueWorker.php => QueueWorker.php} (95%) diff --git a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/QueueWorker.php similarity index 95% rename from public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php rename to public/modules/custom/helfi_annif/src/Plugin/QueueWorker/QueueWorker.php index 371564a03..5b1e9f9ba 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/KeywordQueueWorker.php +++ b/public/modules/custom/helfi_annif/src/Plugin/QueueWorker/QueueWorker.php @@ -23,7 +23,7 @@ * cron = {"time" = 60} * ) */ -final class KeywordQueueWorker extends QueueWorkerBase implements ContainerFactoryPluginInterface { +final class QueueWorker extends QueueWorkerBase implements ContainerFactoryPluginInterface { /** * The keyword manager. From bffb4d436c73ed2e2085cb6f47d55e083f393eab Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:55:09 +0300 Subject: [PATCH 5/9] UHF-10631: Ensure content is not saved when keywords are generated --- .../config/schema/helfi_annif.schema.yml | 8 + .../custom/helfi_annif/helfi_annif.info.yml | 1 + .../custom/helfi_annif/helfi_annif.install | 36 +++ .../custom/helfi_annif/helfi_annif.module | 30 +-- .../helfi_annif/helfi_annif.services.yml | 2 + .../src/Drush/Commands/AnnifCommands.php | 66 ++++- .../src/Entity/SuggestedTopics.php | 7 + .../SuggestedTopicsReferenceItem.php | 78 ++++++ .../SuggestedTopicsReferenceWidget.php | 67 +++++ .../helfi_annif/src/RecommendableBase.php | 40 +-- .../src/RecommendableInterface.php | 8 +- .../helfi_annif/src/ReferenceUpdater.php | 243 ++++++++++++++++++ .../src/SuggestedTopicsInterface.php | 9 + .../custom/helfi_annif/src/TopicsManager.php | 35 +-- .../tests/src/Traits/AnnifApiTestTrait.php | 4 +- 15 files changed, 580 insertions(+), 54 deletions(-) create mode 100644 public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/SuggestedTopicsReferenceItem.php create mode 100644 public/modules/custom/helfi_annif/src/Plugin/Field/FieldWidget/SuggestedTopicsReferenceWidget.php create mode 100644 public/modules/custom/helfi_annif/src/ReferenceUpdater.php diff --git a/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml b/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml index 134c5fb98..aa3367c7c 100644 --- a/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml +++ b/public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml @@ -10,3 +10,11 @@ field.value.helfi_annif_suggestion: score: type: label label: Score + +field.value.suggested_topics_reference: + type: mapping + label: Default value + mapping: + value: + type: label + label: Value diff --git a/public/modules/custom/helfi_annif/helfi_annif.info.yml b/public/modules/custom/helfi_annif/helfi_annif.info.yml index a9357aac1..e6e7404fc 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.info.yml +++ b/public/modules/custom/helfi_annif/helfi_annif.info.yml @@ -9,6 +9,7 @@ dependencies: - helfi_api_base:helfi_api_base - readonly_field_widget:readonly_field_widget - drupal:field + # TODO: remove dependency to etusivu. - helfi_etusivu:helfi_etusivu - helfi_node_news_item:helfi_node_news_item - helfi_node_news_article:helfi_node_news_article diff --git a/public/modules/custom/helfi_annif/helfi_annif.install b/public/modules/custom/helfi_annif/helfi_annif.install index 0beb83b6d..bb7a041f5 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.install +++ b/public/modules/custom/helfi_annif/helfi_annif.install @@ -60,3 +60,39 @@ function helfi_annif_update_9002(): void { \Drupal::entityDefinitionUpdateManager() ->installEntityType($definition); } + +/** + * Updates field storage definition for etusivu news entities. + */ +function helfi_annif_update_9003(): void { + $updates = [ + 'annif_suggested_topics' => [ + 'node' => [ + 'news_article', + 'news_item', + ], + ], + ]; + + $entity_field_manager = \Drupal::service('entity_field.manager'); + $update_manager = \Drupal::entityDefinitionUpdateManager(); + + foreach ($updates as $field => $entity_types) { + foreach ($entity_types as $entity_type_id => $bundles) { + foreach ($bundles as $bundle) { + $field_definitions = $entity_field_manager->getFieldDefinitions($entity_type_id, $bundle); + + $update_manager->installFieldStorageDefinition( + $field, + $entity_type_id, + 'helfi_annif', + $field_definitions[$field], + ); + } + } + } + + \Drupal::messenger()->addMessage('Run drush helfi:annif-fix-references'); + + // @todo remove obsolete field annif_keywords in a future update hook. +} diff --git a/public/modules/custom/helfi_annif/helfi_annif.module b/public/modules/custom/helfi_annif/helfi_annif.module index d5eb328e8..dcdee1e6e 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.module +++ b/public/modules/custom/helfi_annif/helfi_annif.module @@ -159,6 +159,7 @@ function helfi_annif_entity_bundle_field_info_alter(&$fields, EntityTypeInterfac } } + // @todo remove dependency to etusivu config. $recommendable_node_bundles = ['news_item', 'news_article']; if ($entityType->id() == 'node' && in_array($bundle, $recommendable_node_bundles)) { foreach (helfi_annif_bundle_fields($entityType->id(), $bundle) as $name => $field) { @@ -211,30 +212,27 @@ function helfi_annif_bundle_fields(string $entity_type_id, string $bundle): arra ->setDisplayConfigurable('view', TRUE); } - $keywordfieldId = 'annif_keywords'; - $keywordfieldName = new TranslatableMarkup('Automatically selected news categories', [], ['context' => 'annif']); + $fields['annif_suggested_topics'] = BundleFieldDefinition::create('suggested_topics_reference') + ->setName('annif_suggested_topics') + ->setLabel(new TranslatableMarkup('Automatically selected news categories', [], ['context' => 'annif'])) + ->setTargetEntityTypeId($entity_type_id) + ->setTargetBundle($bundle) + ->setReadonly(TRUE) + ->setTranslatable(FALSE); + // @todo remove this. + $keywordfieldId = 'annif_keywords'; $fields[$keywordfieldId] = BundleFieldDefinition::create('entity_reference') ->setName($keywordfieldId) - ->setLabel($keywordfieldName) - ->setSettings(['target_type' => 'taxonomy_term']) ->setTargetEntityTypeId($entity_type_id) ->setTargetBundle($bundle) - ->setDisplayOptions('form', - [ - 'type' => 'readonly_field_widget', - 'third_party_settings' => [], - 'settings' => - [ - 'formatter_type' => 'entity_reference_label', - ], - ] - ) + ->setDisplayOptions('form', ['type' => 'hidden']) + ->setDisplayOptions('view', ['type' => 'hidden']) ->setReadonly(TRUE) ->setTranslatable(FALSE) ->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED) - ->setDisplayConfigurable('form', TRUE) - ->setDisplayConfigurable('view', TRUE); + ->setDisplayConfigurable('form', FALSE) + ->setDisplayConfigurable('view', FALSE); return $fields; } diff --git a/public/modules/custom/helfi_annif/helfi_annif.services.yml b/public/modules/custom/helfi_annif/helfi_annif.services.yml index 2aafb08cb..f9a122bcc 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.services.yml +++ b/public/modules/custom/helfi_annif/helfi_annif.services.yml @@ -9,6 +9,8 @@ services: Drupal\helfi_annif\TopicsManager: ~ + Drupal\helfi_annif\ReferenceUpdater: ~ + Drupal\helfi_annif\RecommendationManager: ~ Drupal\helfi_annif\Client\ApiClient : ~ diff --git a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php index a4592811e..21069b8e3 100644 --- a/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php +++ b/public/modules/custom/helfi_annif/src/Drush/Commands/AnnifCommands.php @@ -10,12 +10,14 @@ use Drupal\Core\Database\Connection; use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\Entity\FieldableEntityInterface; use Drupal\Core\Entity\TranslatableInterface; use Drupal\Core\StringTranslation\StringTranslationTrait; use Drupal\Core\Utility\Error; use Drupal\helfi_annif\Client\ApiClient; -use Drupal\helfi_annif\TopicsManager; +use Drupal\helfi_annif\ReferenceUpdater; use Drupal\helfi_annif\TextConverter\TextConverterManager; +use Drupal\helfi_annif\TopicsManager; use Drush\Attributes\Argument; use Drush\Attributes\Command; use Drush\Attributes\Option; @@ -43,12 +45,15 @@ final class AnnifCommands extends DrushCommands { * The text converter. * @param \Drupal\helfi_annif\TopicsManager $topicsManager * The keyword generator. + * @param \Drupal\helfi_annif\ReferenceUpdater $referenceManager + * The reference manager. */ public function __construct( private readonly Connection $connection, private readonly EntityTypeManagerInterface $entityTypeManager, private readonly TextConverterManager $textConverter, private readonly TopicsManager $topicsManager, + private readonly ReferenceUpdater $referenceManager, ) { parent::__construct(); } @@ -276,4 +281,63 @@ public function batchVisibilityFieldsDefaultValues(array $entityIds, &$context,) } } + /** + * Fix entity references in a batch. + * + * @param array $entityIds + * Ids of entities to update. + * @param array $context + * The context. + */ + public function batchFixEntityReferences(array $entityIds, array &$context): void { + if (!isset($context['sandbox']['from'])) { + $context['sandbox']['from'] = 0; + } + + $batchSize = 50; + $from = $context['sandbox']['from']; + $to = min($from + $batchSize, count($entityIds)); + $slice = array_slice($entityIds, $from, $to - $from); + + try { + foreach ($slice as $item) { + ['entity_type' => $entity_type, 'id' => $id] = $item; + + $entity = $this->entityTypeManager + ->getStorage($entity_type) + ->load($id); + + assert($entity instanceof FieldableEntityInterface); + $this->referenceManager->updateEntityReferenceFields($entity); + } + + $context['sandbox']['from'] = $to; + $context['message'] = sprintf("%d entities remaining", count($entityIds) - $to); + $context['finished'] = $to >= count($entityIds); + } + catch (\Exception $e) { + $context['message'] = sprintf('An error occurred during processing: %s', $e->getMessage()); + $context['finished'] = 1; + } + } + + /** + * Set new fields' default values. + */ + #[Command(name: 'helfi:annif-fix-references')] + public function fixEntityReferences(): int { + $entities = $this->referenceManager->getReferencesWithoutTarget(); + + $batch = (new BatchBuilder()) + ->addOperation([$this, 'batchFixEntityReferences'], [ + $entities, + ]); + + batch_set($batch->toArray()); + + drush_backend_batch_process(); + + return DrushCommands::EXIT_SUCCESS; + } + } diff --git a/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php index 89229db8d..d03e2dd6f 100644 --- a/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php +++ b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php @@ -68,4 +68,11 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type): a return $fields; } + /** + * {@inheritDoc} + */ + public function hasKeywords(): bool { + return $this->keywords->isEmpty(); + } + } diff --git a/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/SuggestedTopicsReferenceItem.php b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/SuggestedTopicsReferenceItem.php new file mode 100644 index 000000000..cf27bfc5b --- /dev/null +++ b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/SuggestedTopicsReferenceItem.php @@ -0,0 +1,78 @@ + 'suggested_topics', + ] + parent::defaultStorageSettings(); + } + + /** + * {@inheritdoc} + */ + public function storageSettingsForm(array &$form, FormStateInterface $form_state, $has_data): array { + $elements = parent::storageSettingsForm($form, $form_state, $has_data); + + $elements['target_type']['#access'] = FALSE; + + return $elements; + } + + /** + * {@inheritdoc} + */ + public function fieldSettingsForm(array $form, FormStateInterface $form_state): array { + $form = parent::fieldSettingsForm($form, $form_state); + + $form['handler']['#access'] = FALSE; + + return $form; + } + + /** + * {@inheritdoc} + */ + public function preSave() { + if ($this->hasNewEntity()) { + $this->entity->save(); + } + + parent::preSave(); + } + + /** + * {@inheritdoc} + */ + public function delete(): void { + if ($this->entity) { + $this->entity->delete(); + } + } + +} diff --git a/public/modules/custom/helfi_annif/src/Plugin/Field/FieldWidget/SuggestedTopicsReferenceWidget.php b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldWidget/SuggestedTopicsReferenceWidget.php new file mode 100644 index 000000000..6b864d480 --- /dev/null +++ b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldWidget/SuggestedTopicsReferenceWidget.php @@ -0,0 +1,67 @@ +target_id) && $items[$delta]->entity; + + /** @var \Drupal\helfi_annif\Entity\SuggestedTopics $entity */ + $entity = $hasTargetEntity ? $items[$delta]->entity : SuggestedTopics::create([]); + + $element['entity'] = [ + '#type' => 'value', + '#value' => $entity, + ]; + + if ($hasTargetEntity) { + $keywords = []; + foreach ($entity->referencedEntities() as $keyword) { + $keywords[] = $keyword->label(); + } + + $element['keywords'] = [ + '#theme' => 'item_list', + '#items' => $keywords, + ]; + + $element['target_id'] = [ + '#type' => 'value', + '#default_value' => $items[$delta]->target_id, + ]; + } + + return $element; + } + +} diff --git a/public/modules/custom/helfi_annif/src/RecommendableBase.php b/public/modules/custom/helfi_annif/src/RecommendableBase.php index 85ee62871..9640cce31 100644 --- a/public/modules/custom/helfi_annif/src/RecommendableBase.php +++ b/public/modules/custom/helfi_annif/src/RecommendableBase.php @@ -4,7 +4,7 @@ namespace Drupal\helfi_annif; -use Drupal\Core\Cache\Cache; +use Drupal\Core\Field\EntityReferenceFieldItemListInterface; use Drupal\node\Entity\Node; /** @@ -12,15 +12,13 @@ */ abstract class RecommendableBase extends Node implements RecommendableInterface { - protected const string KEYWORDFIELD = 'annif_keywords'; - - protected const string SHOW_RECOMMENDATIONS_BLOCK = 'show_annif_block'; + protected const SHOW_RECOMMENDATIONS_BLOCK = 'show_annif_block'; /** * {@inheritDoc} */ public function isRecommendableContent(): bool { - return !$this->get(self::KEYWORDFIELD)->isEmpty(); + return !$this->get(TopicsManager::TOPICS_FIELD)->isEmpty(); } /** @@ -42,14 +40,16 @@ public function showRecommendationsBlock(): bool { * {@inheritDoc} */ public function hasKeywords(): bool { - return !$this->get(self::KEYWORDFIELD)->isEmpty(); - } + $field = $this->getTopicsField(); - /** - * {@inheritDoc} - */ - public function getKeywordFieldName(): string { - return self::KEYWORDFIELD; + foreach ($field->referencedEntities() as $topics) { + assert($topics instanceof SuggestedTopicsInterface); + if ($topics->hasKeywords()) { + return TRUE; + } + } + + return FALSE; } /** @@ -57,7 +57,7 @@ public function getKeywordFieldName(): string { */ public function getCacheTagsToInvalidate(): array { $parentCacheTags = parent::getCacheTagsToInvalidate(); - if (!$this->hasField(self::getKeywordFieldName())) { + if (!$this->hasField(TopicsManager::TOPICS_FIELD)) { return $parentCacheTags; } @@ -72,13 +72,23 @@ public function getCacheTagsToInvalidate(): array { * Array of cache tags for keywords. */ protected function getKeywordsCacheTags(): array { - $terms = $this->get(self::getKeywordFieldName())->referencedEntities(); + $field = $this->getTopicsField(); $tags = array_map( fn ($term) => $term->getCacheTags(), - $terms + $field->referencedEntities() ); return array_merge(...$tags); } + /** + * {@inheritDoc} + */ + public function getTopicsField(): EntityReferenceFieldItemListInterface { + $field = $this->get(TopicsManager::TOPICS_FIELD); + assert($field instanceof EntityReferenceFieldItemListInterface); + + return $field; + } + } diff --git a/public/modules/custom/helfi_annif/src/RecommendableInterface.php b/public/modules/custom/helfi_annif/src/RecommendableInterface.php index cd2588486..aa4d1135e 100644 --- a/public/modules/custom/helfi_annif/src/RecommendableInterface.php +++ b/public/modules/custom/helfi_annif/src/RecommendableInterface.php @@ -45,11 +45,11 @@ public function showRecommendationsBlock(): bool; public function hasKeywords(): bool; /** - * Get keyword field name. + * Get keyword field. * - * @return string - * Name of the field which holds the keywords. + * @return \Drupal\Core\Field\EntityReferenceFieldItemListInterface + * Field which holds the keywords. */ - public function getKeywordFieldName(): string; + public function getTopicsField(): EntityReferenceFieldItemListInterface; } diff --git a/public/modules/custom/helfi_annif/src/ReferenceUpdater.php b/public/modules/custom/helfi_annif/src/ReferenceUpdater.php new file mode 100644 index 000000000..e62b15ba0 --- /dev/null +++ b/public/modules/custom/helfi_annif/src/ReferenceUpdater.php @@ -0,0 +1,243 @@ +getAllReferenceFields(); + + foreach ($fieldNames as $entityType => $bundles) { + foreach ($bundles as $bundle => $fields) { + $ids = $this->entitiesWithNonexistentFields($entityType, $bundle, $fields); + $result = $this->mergeEntityIds($result, $entityType, $ids); + } + } + + return $result; + } + + /** + * Returns the names of reference fields of the given entity data. + * + * @param string $entityType + * The entity type. + * @param string $bundle + * The bundle. + * + * @return array + * The field names. Empty array if this entity does not contain + * suggested_topics_reference fields. + */ + public function getReferenceFields(string $entityType, string $bundle): array { + return $this->getAllReferenceFields()[$entityType][$bundle] ?? []; + } + + /** + * Returns a map of reference fields per entity and bundle. + * + * @return array + * Array of annif_suggested_topics fields. Structure: + * - entity type: array keyed by bundle. + * - bundle: array of field names. + */ + private function getAllReferenceFields(): array { + if (!is_null($this->referenceFields)) { + return $this->referenceFields; + } + + // EntityFieldManagerInterface::getFieldMapByFieldType() does not work here. + // See: https://www.drupal.org/project/drupal/issues/3045509. + $this->referenceFields = []; + foreach ($this->entityTypeManager->getDefinitions() as $entityTypeId => $entityType) { + if (!$entityType->entityClassImplements(FieldableEntityInterface::class)) { + continue; + } + + foreach ($this->entityTypeBundleInfo->getBundleInfo($entityTypeId) as $bundle => $bundleInfo) { + foreach ($this->entityFieldManager->getFieldDefinitions($entityTypeId, $bundle) as $fieldDefinition) { + if ($fieldDefinition->getType() === 'suggested_topics_reference') { + $this->referenceFields[$entityTypeId][$bundle][] = $fieldDefinition->getName(); + } + } + } + } + + return $this->referenceFields; + } + + /** + * Returns IDs of entities where one or more fields have no value. + * + * @param string $entityType + * The entity type. + * @param string $bundle + * The entity bundle. + * @param array $fields + * The entity reference field names to check. + * + * @return array + * Array of entity IDs. Empty array if none were found. + * + * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException + * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException + */ + private function entitiesWithNonexistentFields(string $entityType, string $bundle, array $fields): array { + $query = $this->entityTypeManager + ->getStorage($entityType) + ->getQuery() + ->accessCheck(FALSE); + + $bundleKey = $this->entityTypeManager + ->getDefinition($entityType) + ->getKey('bundle'); + if ($bundleKey) { + $query->condition($bundleKey, $bundle); + } + + $orGroup = $query->orConditionGroup(); + foreach ($fields as $field) { + $orGroup->notExists($field); + } + $query->condition($orGroup); + + return $query->execute(); + } + + /** + * Merges entity IDs of multiple entity types. + * + * @param array $data + * The existing entity IDs. + * @param string $entityType + * The entity type of the IDs. + * @param array $ids + * The IDs to merge. + * + * @return array + * The merged data. Associative array keyed by "entity_type:entity_id". + * Values a structured array: + * - entity_type: The entity type. + * - id: The entity ID. + */ + private function mergeEntityIds(array $data, string $entityType, array $ids): array { + foreach ($ids as $id) { + $data["$entityType:$id"] = [ + 'entity_type' => $entityType, + 'id' => $id, + ]; + } + + return $data; + } + + /** + * Adds missing annif entities to reference fields. + * + * @param \Drupal\Core\Entity\FieldableEntityInterface $entity + * The entity to update. + * + * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException + * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException + * @throws \Drupal\Core\Entity\EntityStorageException + * @throws \Drupal\Core\TypedData\Exception\ReadOnlyException + */ + public function updateEntityReferenceFields(FieldableEntityInterface $entity): void { + $entityIsUpdated = FALSE; + $fieldNames = $this->getReferenceFields($entity->getEntityTypeId(), $entity->bundle()); + + foreach ($fieldNames as $fieldName) { + $referenceField = $entity->get($fieldName); + if ($referenceField->isEmpty()) { + $topicsEntity = $this->entityTypeManager + ->getStorage('suggested_topics') + ->create([]); + + $referenceField->setValue($topicsEntity); + + assert($topicsEntity instanceof SuggestedTopics); + $this->copyLegacyData($entity, $topicsEntity); + + $entityIsUpdated = TRUE; + } + } + + if ($entityIsUpdated) { + $entity->save(); + } + } + + /** + * Copy values from legacy field to the new entity. + * + * @todo remove this once the legacy field is deleted. + */ + private function copyLegacyData(FieldableEntityInterface $entity, SuggestedTopicsInterface $topics): void { + if (!$entity->hasField('annif_keywords')) { + return; + } + + $keywords = $entity->get('annif_keywords')->getValue(); + $count = count($keywords); + $keywords = array_map(fn (array $item, int $index) => $item + [ + // The legacy data does not have scores. However, the keywords + // are sorted by score, so this maps the index to range [0, 0.5]. + 'score' => ($count - $index) / $count / 2, + ], $keywords, array_keys($keywords)); + + $topics->set('keywords', $keywords); + } + +} diff --git a/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php b/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php index 4288a4165..15d8ce02b 100644 --- a/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php +++ b/public/modules/custom/helfi_annif/src/SuggestedTopicsInterface.php @@ -10,4 +10,13 @@ * Provides an interface defining a suggestted topics entity type. */ interface SuggestedTopicsInterface extends ContentEntityInterface { + + /** + * Check if the entity has keywords. + * + * @return bool + * Entity has keywords. + */ + public function hasKeywords(): bool; + } diff --git a/public/modules/custom/helfi_annif/src/TopicsManager.php b/public/modules/custom/helfi_annif/src/TopicsManager.php index 300e8e6e8..4db6ad2a5 100644 --- a/public/modules/custom/helfi_annif/src/TopicsManager.php +++ b/public/modules/custom/helfi_annif/src/TopicsManager.php @@ -8,6 +8,7 @@ use Drupal\Core\Entity\EntityStorageInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Entity\TranslatableInterface; +use Drupal\Core\Field\EntityReferenceFieldItemListInterface; use Drupal\Core\Queue\QueueFactory; use Drupal\helfi_annif\Client\ApiClient; use Drupal\helfi_annif\Client\Keyword; @@ -17,7 +18,7 @@ */ final class TopicsManager { - public const KEYWORD_FIELD = 'annif_keywords'; + public const TOPICS_FIELD = 'annif_suggested_topics'; public const KEYWORD_VID = 'annif_keywords'; /** @@ -88,7 +89,7 @@ private function isEntityProcessed(EntityInterface $entity) : bool { */ public function queueEntity(RecommendableInterface $entity, bool $overwriteExisting = FALSE) : void { if ( - !$entity->hasField(self::KEYWORD_FIELD) || + !$entity->hasField(self::TOPICS_FIELD) || // Skip if entity was processed in this request. $this->isEntityProcessed($entity) || // Skip if entity already has keywords. @@ -111,7 +112,7 @@ public function queueEntity(RecommendableInterface $entity, bool $overwriteExist * Generates keywords for single entity. * * @param \Drupal\helfi_annif\RecommendableInterface $entity - * The entities. + * The entity. * @param bool $overwriteExisting * Overwrites existing keywords when set to TRUE. * @@ -119,7 +120,7 @@ public function queueEntity(RecommendableInterface $entity, bool $overwriteExist * @throws \Drupal\Core\Entity\EntityStorageException */ public function processEntity(RecommendableInterface $entity, bool $overwriteExisting = FALSE) : void { - if (!$entity->hasField(self::KEYWORD_FIELD)) { + if (!$entity->hasField(self::TOPICS_FIELD)) { return; } @@ -181,8 +182,7 @@ private function prepareBatches(array $entities, bool $overwriteExisting) : \Gen foreach ($entities as $key => $entity) { assert($entity instanceof RecommendableInterface); - - if (!$entity->hasField(self::KEYWORD_FIELD)) { + if (!$entity->hasField(self::TOPICS_FIELD)) { continue; } @@ -219,19 +219,22 @@ private function prepareBatches(array $entities, bool $overwriteExisting) : \Gen * @throws \Drupal\Core\Entity\EntityStorageException */ private function saveKeywords(RecommendableInterface $entity, array $keywords) : void { - $terms = []; - - foreach ($keywords as $keyword) { - $terms[] = $this->getTerm($keyword, $entity->language()->getId()); + $values = array_map(fn($keyword) => [ + 'entity' => $this->getTerm($keyword, $entity->language()->getId()), + 'score' => $keyword->score, + ], $keywords); + + $field = $entity->get(self::TOPICS_FIELD); + assert($field instanceof EntityReferenceFieldItemListInterface); + foreach ($field->referencedEntities() as $topicsEntity) { + /** @var \Drupal\helfi_annif\SuggestedTopicsInterface $topicsEntity */ + $topicsEntity->set('keywords', $values); + $topicsEntity->save(); } - $entity->set($entity->getKeywordFieldName(), $terms); - - // This needs to be before ->save() so - // processedItems is set for update hooks. + // Mark as processed so the same entity is bombarding the + // API if it is queued multiple times for some reason. $this->processedItems[$this->getEntityKey($entity)] = TRUE; - - $entity->save(); } /** diff --git a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php index 9e174e35e..a4e60bf0d 100644 --- a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php +++ b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php @@ -81,10 +81,10 @@ protected function mockEntity(string $langcode = 'fi', bool|NULL $hasKeywords = $field->isEmpty()->willReturn(!$hasKeywords); $entity - ->hasField(Argument::exact(TopicsManager::KEYWORD_FIELD)) + ->hasField(Argument::exact(TopicsManager::TOPICS_FIELD)) ->willReturn($hasKeywords !== NULL); $entity - ->get(Argument::exact(TopicsManager::KEYWORD_FIELD)) + ->get(Argument::exact(TopicsManager::TOPICS_FIELD)) ->willReturn($field->reveal()); if (is_bool($shouldSave)) { From 5f3e609b132995321c1b103ef6e1dde5f3e82734 Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:55:13 +0300 Subject: [PATCH 6/9] UHF-10631: Fix recommendations block and tests --- .../custom/helfi_annif/helfi_annif.module | 6 +-- .../src/Entity/SuggestedTopics.php | 31 +++++++++++- .../src/Plugin/Block/RecommendationsBlock.php | 33 ++++++++----- .../helfi_annif/src/RecommendableBase.php | 33 +------------ .../src/RecommendableInterface.php | 8 --- .../helfi_annif/src/RecommendationManager.php | 49 ++++++++++--------- .../tests/src/Kernel/KeywordManagerTest.php | 1 + .../tests/src/Traits/AnnifApiTestTrait.php | 31 ++++++------ .../tests/src/Unit/Client/ApiClientTest.php | 2 +- .../tests/src/Unit/KeywordManagerTest.php | 2 +- 10 files changed, 99 insertions(+), 97 deletions(-) diff --git a/public/modules/custom/helfi_annif/helfi_annif.module b/public/modules/custom/helfi_annif/helfi_annif.module index dcdee1e6e..d577f9ed8 100644 --- a/public/modules/custom/helfi_annif/helfi_annif.module +++ b/public/modules/custom/helfi_annif/helfi_annif.module @@ -218,7 +218,9 @@ function helfi_annif_bundle_fields(string $entity_type_id, string $bundle): arra ->setTargetEntityTypeId($entity_type_id) ->setTargetBundle($bundle) ->setReadonly(TRUE) - ->setTranslatable(FALSE); + ->setTranslatable(FALSE) + ->setDisplayOptions('form', ['type' => 'suggested_topics_reference']) + ->setDisplayConfigurable('form', TRUE); // @todo remove this. $keywordfieldId = 'annif_keywords'; @@ -226,8 +228,6 @@ function helfi_annif_bundle_fields(string $entity_type_id, string $bundle): arra ->setName($keywordfieldId) ->setTargetEntityTypeId($entity_type_id) ->setTargetBundle($bundle) - ->setDisplayOptions('form', ['type' => 'hidden']) - ->setDisplayOptions('view', ['type' => 'hidden']) ->setReadonly(TRUE) ->setTranslatable(FALSE) ->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED) diff --git a/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php index d03e2dd6f..c48bc0043 100644 --- a/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php +++ b/public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php @@ -4,9 +4,11 @@ namespace Drupal\helfi_annif\Entity; +use Drupal\Core\Cache\Cache; use Drupal\Core\Entity\ContentEntityBase; use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Field\BaseFieldDefinition; +use Drupal\Core\Field\EntityReferenceFieldItemListInterface; use Drupal\Core\Field\FieldStorageDefinitionInterface; use Drupal\Core\StringTranslation\TranslatableMarkup; use Drupal\helfi_annif\SuggestedTopicsInterface; @@ -32,7 +34,7 @@ * }, * ) */ -final class SuggestedTopics extends ContentEntityBase implements SuggestedTopicsInterface { +class SuggestedTopics extends ContentEntityBase implements SuggestedTopicsInterface { /** * {@inheritdoc} @@ -72,7 +74,32 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type): a * {@inheritDoc} */ public function hasKeywords(): bool { - return $this->keywords->isEmpty(); + return $this->get('keywords')->isEmpty(); + } + + /** + * {@inheritDoc} + */ + public function getCacheTagsToInvalidate(): array { + return Cache::mergeTags(parent::getCacheTagsToInvalidate(), ...$this->getKeywordsCacheTags()); + } + + /** + * Get the cache tags for all the keywords. + * + * @return array + * Array of cache tags for keywords. + */ + protected function getKeywordsCacheTags(): array { + $tags = []; + + $field = $this->get('keywords'); + assert($field instanceof EntityReferenceFieldItemListInterface); + foreach ($field->referencedEntities() as $keyword) { + $tags[] = $keyword->getCacheTags(); + } + + return $tags; } } diff --git a/public/modules/custom/helfi_annif/src/Plugin/Block/RecommendationsBlock.php b/public/modules/custom/helfi_annif/src/Plugin/Block/RecommendationsBlock.php index 3db2e5803..ece770d27 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/Block/RecommendationsBlock.php +++ b/public/modules/custom/helfi_annif/src/Plugin/Block/RecommendationsBlock.php @@ -9,13 +9,16 @@ use Drupal\Core\Block\BlockBase; use Drupal\Core\Cache\Cache; use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\Field\EntityReferenceFieldItemListInterface; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Plugin\Context\EntityContextDefinition; use Drupal\Core\Plugin\ContextAwarePluginInterface; use Drupal\Core\Session\AccountInterface; use Drupal\Core\StringTranslation\TranslatableMarkup; +use Drupal\Core\Utility\Error; use Drupal\helfi_annif\RecommendableInterface; use Drupal\helfi_annif\RecommendationManager; +use Drupal\helfi_annif\TopicsManager; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -50,8 +53,8 @@ public function __construct( /** * {@inheritdoc} */ - public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) : static { - return new static($configuration, $plugin_id, $plugin_definition, + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) : self { + return new self($configuration, $plugin_id, $plugin_definition, $container->get(RecommendationManager::class), $container->get('entity_type.manager'), $container->get('current_user'), @@ -67,7 +70,7 @@ public function build() : array { $node = $this->getContextValue('node'); } catch (ContextException $exception) { - $this->logger->error($exception->getMessage()); + Error::logException($this->logger, $exception); return []; } @@ -90,14 +93,17 @@ public function build() : array { return $response; } - $nodes = []; + $view_builder = $this->entityTypeManager->getViewBuilder('node'); + // We want to render the recommendation results as nodes // so that all fields are correctly preprocessed. + $nodes = []; foreach ($recommendations as $recommendation) { - $view_builder = $this->entityTypeManager->getViewBuilder('node'); $nodes[] = $view_builder->view($recommendation, 'teaser'); } + $response['#rows'] = $nodes; + return $response; } @@ -107,7 +113,7 @@ public function build() : array { public function getCacheContexts(): array { return Cache::mergeContexts( parent::getCacheContexts(), - ['languages:language_content', 'user.roles:anonymous'], + ['languages:language_content', 'user.roles:anonymous', 'url.path'], ); } @@ -117,10 +123,15 @@ public function getCacheContexts(): array { public function getCacheTags(): array { $node = $this->getContextValue('node'); - return Cache::mergeTags( - parent::getCacheTags(), - $node->getCacheTags(), - ); + $topics = $node->get(TopicsManager::TOPICS_FIELD); + assert($topics instanceof EntityReferenceFieldItemListInterface); + + $tags = []; + foreach ($topics->referencedEntities() as $entity) { + $tags[] = $entity->getCacheTags(); + } + + return Cache::mergeTags(parent::getCacheTags(), ...$tags); } /** @@ -138,7 +149,7 @@ private function getRecommendations(RecommendableInterface $node): array { ->getRecommendations($node, 3, 'fi'); } catch (\Exception $exception) { - $this->logger->error($exception->getMessage()); + Error::logException($this->logger, $exception); return []; } return $recommendations; diff --git a/public/modules/custom/helfi_annif/src/RecommendableBase.php b/public/modules/custom/helfi_annif/src/RecommendableBase.php index 9640cce31..d507957b8 100644 --- a/public/modules/custom/helfi_annif/src/RecommendableBase.php +++ b/public/modules/custom/helfi_annif/src/RecommendableBase.php @@ -53,38 +53,9 @@ public function hasKeywords(): bool { } /** - * {@inheritDoc} - */ - public function getCacheTagsToInvalidate(): array { - $parentCacheTags = parent::getCacheTagsToInvalidate(); - if (!$this->hasField(TopicsManager::TOPICS_FIELD)) { - return $parentCacheTags; - } - - $keywordsCacheTags = $this->getKeywordsCacheTags(); - return Cache::mergeTags($parentCacheTags, $keywordsCacheTags); - } - - /** - * Get the cache tags for all the keywords. - * - * @return array - * Array of cache tags for keywords. - */ - protected function getKeywordsCacheTags(): array { - $field = $this->getTopicsField(); - - $tags = array_map( - fn ($term) => $term->getCacheTags(), - $field->referencedEntities() - ); - return array_merge(...$tags); - } - - /** - * {@inheritDoc} + * Get topics field. */ - public function getTopicsField(): EntityReferenceFieldItemListInterface { + private function getTopicsField(): EntityReferenceFieldItemListInterface { $field = $this->get(TopicsManager::TOPICS_FIELD); assert($field instanceof EntityReferenceFieldItemListInterface); diff --git a/public/modules/custom/helfi_annif/src/RecommendableInterface.php b/public/modules/custom/helfi_annif/src/RecommendableInterface.php index aa4d1135e..d900c8b90 100644 --- a/public/modules/custom/helfi_annif/src/RecommendableInterface.php +++ b/public/modules/custom/helfi_annif/src/RecommendableInterface.php @@ -44,12 +44,4 @@ public function showRecommendationsBlock(): bool; */ public function hasKeywords(): bool; - /** - * Get keyword field. - * - * @return \Drupal\Core\Field\EntityReferenceFieldItemListInterface - * Field which holds the keywords. - */ - public function getTopicsField(): EntityReferenceFieldItemListInterface; - } diff --git a/public/modules/custom/helfi_annif/src/RecommendationManager.php b/public/modules/custom/helfi_annif/src/RecommendationManager.php index f23accded..8e0310886 100644 --- a/public/modules/custom/helfi_annif/src/RecommendationManager.php +++ b/public/modules/custom/helfi_annif/src/RecommendationManager.php @@ -89,34 +89,37 @@ public function getRecommendations(EntityInterface $entity, int $limit = 3, stri * @return array * Database query result. */ - private function executeQuery(EntityInterface $entity, string $target_langcode, string $destination_langcode, int $limit) { + private function executeQuery(EntityInterface $entity, string $target_langcode, string $destination_langcode, int $limit): array { $query = " - select + SELECT n.nid, count(n.nid) as relevancy, nfd.created, nfd.status - from node as n - left join node__annif_keywords as annif on n.nid = annif.entity_id - left join node_field_data as nfd on nfd.nid = n.nid - where annif.annif_keywords_target_id in - (select annif_keywords_target_id - from node__annif_keywords - where entity_id = :nid and - langcode = :target_langcode) - and n.nid not in - (select distinct restriction.entity_id - from node__in_recommendations as restriction - where restriction.in_recommendations_value = 0) - and nfd.status = 1 - and n.langcode = :target_langcode - and annif.langcode = :target_langcode - and nfd.langcode = :destination_langcode - and n.nid != :nid - and nfd.created > :timestamp - group by n.nid - order by count(n.nid) DESC - limit {$limit}; + FROM node as n + INNER JOIN node__annif_suggested_topics as reference ON n.nid = reference.entity_id + INNER JOIN suggested_topics as topics ON topics.id = reference.annif_suggested_topics_target_id + INNER JOIN suggested_topics__keywords as keywords ON topics.id = keywords.entity_id + INNER JOIN node_field_data as nfd ON nfd.nid = n.nid AND nfd.langcode = :destination_langcode + WHERE + nfd.status = 1 + -- Select rows that have keywords in common with current entity. + AND keywords.keywords_target_id IN + (SELECT keywords_target_id + FROM suggested_topics__keywords stk + INNER JOIN node__annif_suggested_topics AS node ON stk.entity_id = node.annif_suggested_topics_target_id + WHERE node.entity_id = :nid) + -- Filter out entities that should be hidden from recommendations. + AND n.nid NOT IN + (SELECT DISTINCT restriction.entity_id + FROM node__in_recommendations as restriction + WHERE restriction.in_recommendations_value = 0) + AND n.langcode = :target_langcode + AND n.nid != :nid + AND nfd.created > :timestamp + GROUP BY n.nid + ORDER BY count(n.nid) DESC + LIMIT {$limit}; "; // Cannot add :limit as parameter here, diff --git a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php index 247fe00fd..308080779 100644 --- a/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Kernel/KeywordManagerTest.php @@ -46,6 +46,7 @@ public function setUp(): void { $entities = [ 'taxonomy_term', + 'suggested_topics', ]; foreach ($entities as $entity) { diff --git a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php index a4e60bf0d..2d2542139 100644 --- a/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php +++ b/public/modules/custom/helfi_annif/tests/src/Traits/AnnifApiTestTrait.php @@ -4,8 +4,9 @@ namespace Drupal\Tests\helfi_annif\Traits; -use Drupal\Core\Field\FieldItemListInterface; +use Drupal\Core\Field\EntityReferenceFieldItemListInterface; use Drupal\Core\Language\LanguageInterface; +use Drupal\helfi_annif\Entity\SuggestedTopics; use Drupal\helfi_annif\RecommendableInterface; use Drupal\helfi_annif\TextConverter\TextConverterInterface; use Drupal\helfi_annif\TextConverter\TextConverterManager; @@ -52,10 +53,8 @@ protected function getFixture(string $name): string { * Value for keyword field ->isEmpty(), NULL for ->hasField() = FALSE. * @param bool|null $shouldSave * Bool if $entity->save() should be called, NULL for no opinion. - * @param bool $hasKeywordField - * Bool if entity should have keyword field. */ - protected function mockEntity(string $langcode = 'fi', bool|NULL $hasKeywords = FALSE, bool|NULL $shouldSave = NULL, $hasKeywordField = TRUE): RecommendableInterface { + protected function mockEntity(string $langcode = 'fi', bool|NULL $hasKeywords = FALSE, bool|NULL $shouldSave = NULL): RecommendableInterface { $language = $this->prophesize(LanguageInterface::class); $language ->getId() @@ -66,34 +65,32 @@ protected function mockEntity(string $langcode = 'fi', bool|NULL $hasKeywords = ->language() ->willReturn($language->reveal()); - $entity->hasField('annif_keywords')->willReturn($hasKeywordField); - - $entity->hasKeywords()->willReturn($hasKeywords ?? FALSE); - - $entity->hasField('annif_keywords')->willReturn(TRUE); - $entity->getKeywordFieldName()->willReturn('annif_keywords'); - $entity->getEntityTypeId()->willReturn('test_entity'); $entity->bundle()->willReturn('test_entity'); $entity->id()->willReturn($this->randomString()); - $field = $this->prophesize(FieldItemListInterface::class); - $field->isEmpty()->willReturn(!$hasKeywords); - $entity ->hasField(Argument::exact(TopicsManager::TOPICS_FIELD)) ->willReturn($hasKeywords !== NULL); + $entity->hasKeywords()->willReturn($hasKeywords ?? FALSE); + + $topicsEntity = $this->prophesize(SuggestedTopics::class); + + $field = $this->prophesize(EntityReferenceFieldItemListInterface::class); + $field->isEmpty()->willReturn(!$hasKeywords); + $field->referencedEntities()->willReturn([$topicsEntity->reveal()]); + $entity ->get(Argument::exact(TopicsManager::TOPICS_FIELD)) ->willReturn($field->reveal()); if (is_bool($shouldSave)) { if ($shouldSave) { - $entity->set(Argument::any(), Argument::any())->shouldBeCalled(); - $entity->save()->shouldBeCalled(); + $topicsEntity->set(Argument::any(), Argument::any())->shouldBeCalled(); + $topicsEntity->save()->shouldBeCalled(); } else { - $entity->save()->shouldNotBeCalled(); + $topicsEntity->save()->shouldNotBeCalled(); } } diff --git a/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php b/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php index e20374084..fea4599b5 100644 --- a/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Unit/Client/ApiClientTest.php @@ -22,7 +22,7 @@ * * @group helfi_annif */ -class KeywordClientTest extends UnitTestCase { +class ApiClientTest extends UnitTestCase { use AnnifApiTestTrait; diff --git a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php index 65c6a76a7..92bd8a56d 100644 --- a/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php +++ b/public/modules/custom/helfi_annif/tests/src/Unit/KeywordManagerTest.php @@ -29,7 +29,7 @@ class KeywordManagerTest extends UnitTestCase { */ public function testUnsupportedEntity(): void { // hasField(TopicsManager::KEYWORD_FIELD) for entity is FALSE. - $entity = $this->mockEntity(hasKeywords: NULL, hasKeywordField: FALSE); + $entity = $this->mockEntity(hasKeywords: NULL); $queue = $this->prophesize(QueueInterface::class); $queue ->createItem(Argument::any()) From 0c338ed7f76b944ee65c99149c926d86bb36c32d Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 08:55:16 +0300 Subject: [PATCH 7/9] UHF-10631: Configure form displays --- ...form_display.node.news_article.default.yml | 24 ++++++------------ ...ty_form_display.node.news_item.default.yml | 25 ++++++------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/conf/cmi/core.entity_form_display.node.news_article.default.yml b/conf/cmi/core.entity_form_display.node.news_article.default.yml index ac8584095..13084b60a 100644 --- a/conf/cmi/core.entity_form_display.node.news_article.default.yml +++ b/conf/cmi/core.entity_form_display.node.news_article.default.yml @@ -19,13 +19,13 @@ dependencies: module: - field_group - hdbt_admin_tools + - helfi_annif - linkit - media_library - paragraphs - path - publication_date - radioactivity - - readonly_field_widget - scheduler - select2 third_party_settings: @@ -78,9 +78,9 @@ third_party_settings: required_fields: true group_automatically_recommended: children: + - annif_suggested_topics - in_recommendations - show_annif_block - - annif_keywords label: 'Automatically recommended content' region: content parent_name: '' @@ -99,19 +99,11 @@ targetEntityType: node bundle: news_article mode: default content: - annif_keywords: - type: readonly_field_widget - weight: 30 + annif_suggested_topics: + type: suggested_topics_reference + weight: 31 region: content - settings: - label: above - formatter_type: entity_reference_label - formatter_settings: - entity_reference_entity_view: - view_mode: default - entity_reference_label: - link: false - show_description: false + settings: { } third_party_settings: { } created: type: datetime_timestamp @@ -231,7 +223,7 @@ content: third_party_settings: { } in_recommendations: type: boolean_checkbox - weight: 28 + weight: 32 region: content settings: display_label: true @@ -275,7 +267,7 @@ content: third_party_settings: { } show_annif_block: type: boolean_checkbox - weight: 29 + weight: 33 region: content settings: display_label: true diff --git a/conf/cmi/core.entity_form_display.node.news_item.default.yml b/conf/cmi/core.entity_form_display.node.news_item.default.yml index 88d63a300..2ff8ed02a 100644 --- a/conf/cmi/core.entity_form_display.node.news_item.default.yml +++ b/conf/cmi/core.entity_form_display.node.news_item.default.yml @@ -19,13 +19,13 @@ dependencies: module: - field_group - hdbt_admin_tools + - helfi_annif - linkit - media_library - paragraphs - path - publication_date - radioactivity - - readonly_field_widget - scheduler - select2 third_party_settings: @@ -79,7 +79,6 @@ third_party_settings: group_updating_news: children: - field_news_item_updating_news - - field_news_item_updating_news label: 'Updating news' region: content parent_name: '' @@ -94,9 +93,9 @@ third_party_settings: required_fields: false group_automatically_recommended: children: + - annif_suggested_topics - in_recommendations - show_annif_block - - annif_keywords label: 'Automatically recommended content' region: content parent_name: '' @@ -115,19 +114,11 @@ targetEntityType: node bundle: news_item mode: default content: - annif_keywords: - type: readonly_field_widget - weight: 2 + annif_suggested_topics: + type: suggested_topics_reference + weight: 29 region: content - settings: - label: above - formatter_type: entity_reference_label - formatter_settings: - entity_reference_entity_view: - view_mode: default - entity_reference_label: - link: false - show_description: false + settings: { } third_party_settings: { } created: type: datetime_timestamp @@ -261,7 +252,7 @@ content: third_party_settings: { } in_recommendations: type: boolean_checkbox - weight: 0 + weight: 30 region: content settings: display_label: true @@ -305,7 +296,7 @@ content: third_party_settings: { } show_annif_block: type: boolean_checkbox - weight: 1 + weight: 31 region: content settings: display_label: true From 980d723e4fcb9361686e0e78b32f4bbd1985d89a Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 09:13:10 +0300 Subject: [PATCH 8/9] UHF-10631: Replace random function to pass Sonarcloud --- .../src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php index fbdecbad6..61e264821 100644 --- a/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php +++ b/public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php @@ -62,7 +62,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) public static function generateSampleValue(FieldDefinitionInterface $field_definition): array { $values = parent::generateSampleValue($field_definition); - $values['score'] = mt_rand() / mt_getrandmax(); + $values['score'] = random_int(0, 100) / 100; return $values; } From 4010966df2dba8791826d5c290c5c8f76700a08b Mon Sep 17 00:00:00 2001 From: Santeri Hurnanen Date: Tue, 24 Sep 2024 09:44:52 +0300 Subject: [PATCH 9/9] UHF-10631: Ignore GHSA-mg8j-w93w-xjgc --- composer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e47af5f5e..ad1f32df6 100644 --- a/composer.json +++ b/composer.json @@ -69,7 +69,8 @@ "php-http/discovery": false }, "audit": { - "abandoned": "report" + "abandoned": "report", + "ignore": ["GHSA-mg8j-w93w-xjgc"] } }, "extra": {