From d34b8c6a99a289dddbc2bb64c86266c9a6164cb6 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 29 Aug 2024 13:22:14 +0300 Subject: [PATCH 01/57] UHF-8706: correct comment --- .../custom/helfi_rekry_content/src/Entity/JobListing.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_rekry_content/src/Entity/JobListing.php b/public/modules/custom/helfi_rekry_content/src/Entity/JobListing.php index f4e5efd1..f466a7bb 100644 --- a/public/modules/custom/helfi_rekry_content/src/Entity/JobListing.php +++ b/public/modules/custom/helfi_rekry_content/src/Entity/JobListing.php @@ -7,7 +7,7 @@ use Drupal\node\Entity\Node; /** - * Bundle class for hel_map paragraph. + * Bundle class for JobListing paragraph. */ class JobListing extends Node { From 5000ad23641f879431691e427adb78f0f7e69a8f Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 29 Aug 2024 13:23:40 +0300 Subject: [PATCH 02/57] UHF-8706: initialize helfi_google_api module --- .../custom/helfi_google_api/helfi_google_api.info.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 public/modules/custom/helfi_google_api/helfi_google_api.info.yml diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml new file mode 100644 index 00000000..93d13261 --- /dev/null +++ b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml @@ -0,0 +1,7 @@ +name: 'Helfi Google api' +type: module +description: 'Google indexing api integration' +package: HELfi +core_version_requirement: ^10 || ^11 +dependencies: + - 'publication_date:publication_date' From db8f8e42f8b0faa82080318b000ab7dbc5fc6716 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 29 Aug 2024 14:42:07 +0300 Subject: [PATCH 03/57] UHF-8706: initial module code --- conf/cmi/core.extension.yml | 1 + .../helfi_google_api.services.yml | 13 ++++++ .../src/Drush/Commands/HelfiApiCommands.php | 31 ++++++++++++++ .../helfi_google_api/src/HelfiGoogleApi.php | 34 +++++++++++++++ .../src/JobIndexingService.php | 41 +++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 public/modules/custom/helfi_google_api/helfi_google_api.services.yml create mode 100644 public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php create mode 100644 public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php create mode 100644 public/modules/custom/helfi_google_api/src/JobIndexingService.php diff --git a/conf/cmi/core.extension.yml b/conf/cmi/core.extension.yml index c1ad84e9..13281915 100644 --- a/conf/cmi/core.extension.yml +++ b/conf/cmi/core.extension.yml @@ -46,6 +46,7 @@ module: helfi_ckeditor: 0 helfi_etusivu_entities: 0 helfi_eu_cookie_compliance: 0 + helfi_google_api: 0 helfi_hakuvahti: 0 helfi_image_styles: 0 helfi_media: 0 diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml new file mode 100644 index 00000000..247d728e --- /dev/null +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -0,0 +1,13 @@ +services: + _defaults: + autoconfigure: true + autowire: true + + logger.channel.helfi_google_api: + parent: logger.channel_base + arguments: ['helfi_google_api'] + + Drupal\helfi_google_api\HelfiGoogleApi: ~ + + Drupal\helfi_google_api\JobIndexingService: ~ + diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php new file mode 100644 index 00000000..012c0825 --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -0,0 +1,31 @@ +jobIndexingService->indexUpdatedJobs(); + return DrushCommands::EXIT_SUCCESS; + } + +} diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php new file mode 100644 index 00000000..af177fc1 --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -0,0 +1,34 @@ +setApplicationName('Helfi_Rekry'); + //$indexing = new UrlNotification(); + // $apiClient->setDeveloperKey(); + // $apiClient->setAuthConfig(); + // $this->apiClient->setUseBatch(TRUE); + // $batch = $this->apiClient->execute(); + + // $apiClient->setScopes(self::SCOPES); + } + + public function indexJobs(array $jobs, string $type): void { + } + +} diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php new file mode 100644 index 00000000..30cb512c --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -0,0 +1,41 @@ +entityTypeManager + ->getStorage('job_listing'); + $startOfToday = strtotime('today 00:05'); + $startOfYesterday = strtotime('yesterday 00:05'); + + $this->helfiGoogleApi->indexJobs(); + $publishedJobIds = $storage->getQuery() + ->condition('publish_on', $startOfToday, '>=') + ->condition('status', 1) + ->accessCheck(FALSE) + ->execute(); + $publishedJobs = $storage->loadMultiple($publishedJobIds); + + $this->helfiGoogleApi->indexJobs($publishedJobs, 'update'); + + } + +} From 8bebe497a652fef758448b2d05c358868288cf93 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 4 Sep 2024 09:55:22 +0300 Subject: [PATCH 04/57] UHF-8706: add google's api library --- composer.json | 11 +- composer.lock | 285 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 292 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index ce96ab23..878ac438 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,8 @@ "drupal/migrate_plus": "^6.0", "drupal/raven": "^5.0", "drupal/redis": "^1.5", - "drush/drush": "^12" + "drush/drush": "^12", + "google/apiclient": "^2.17" }, "require-dev": { "donatj/mock-webserver": "^2.4", @@ -99,7 +100,10 @@ }, "patchLevel": { "drupal/core": "-p2" - } + }, + "google/apiclient-services": [ + "Indexing" + ] }, "repositories": [ { @@ -121,6 +125,7 @@ "copy-commit-message-script": "make copy-commit-message-script", "post-install-cmd": [ "@copy-commit-message-script" - ] + ], + "pre-autoload-dump": "Google\\Task\\Composer::cleanup" } } diff --git a/composer.lock b/composer.lock index 417730c1..f11d3d8f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5289a22b79ddfac67b8b4ab51b94ab68", + "content-hash": "e1ab31840c7db8f1728b13d29a396d38", "packages": [ { "name": "asm89/stack-cors", @@ -8369,6 +8369,179 @@ }, "time": "2022-08-18T13:55:30+00:00" }, + { + "name": "google/apiclient", + "version": "v2.17.0", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-api-php-client.git", + "reference": "b1f63d72c44307ec8ef7bf18f1012de35d8944ed" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-api-php-client/zipball/b1f63d72c44307ec8ef7bf18f1012de35d8944ed", + "reference": "b1f63d72c44307ec8ef7bf18f1012de35d8944ed", + "shasum": "" + }, + "require": { + "firebase/php-jwt": "^6.0", + "google/apiclient-services": "~0.350", + "google/auth": "^1.37", + "guzzlehttp/guzzle": "^7.4.5", + "guzzlehttp/psr7": "^2.6", + "monolog/monolog": "^2.9||^3.0", + "php": "^8.0", + "phpseclib/phpseclib": "^3.0.36" + }, + "require-dev": { + "cache/filesystem-adapter": "^1.1", + "composer/composer": "^1.10.23", + "phpcompatibility/php-compatibility": "^9.2", + "phpspec/prophecy-phpunit": "^2.1", + "phpunit/phpunit": "^9.6", + "squizlabs/php_codesniffer": "^3.8", + "symfony/css-selector": "~2.1", + "symfony/dom-crawler": "~2.1" + }, + "suggest": { + "cache/filesystem-adapter": "For caching certs and tokens (using Google\\Client::setCache)" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.x-dev" + } + }, + "autoload": { + "files": [ + "src/aliases.php" + ], + "psr-4": { + "Google\\": "src/" + }, + "classmap": [ + "src/aliases.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Client library for Google APIs", + "homepage": "http://developers.google.com/api-client-library/php", + "keywords": [ + "google" + ], + "support": { + "issues": "https://github.com/googleapis/google-api-php-client/issues", + "source": "https://github.com/googleapis/google-api-php-client/tree/v2.17.0" + }, + "time": "2024-07-10T14:57:54+00:00" + }, + { + "name": "google/apiclient-services", + "version": "v0.370.0", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-api-php-client-services.git", + "reference": "25ad8515701dd832313d0f5f0a828670d60e541a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/25ad8515701dd832313d0f5f0a828670d60e541a", + "reference": "25ad8515701dd832313d0f5f0a828670d60e541a", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.6" + }, + "type": "library", + "autoload": { + "files": [ + "autoload.php" + ], + "psr-4": { + "Google\\Service\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Client library for Google APIs", + "homepage": "http://developers.google.com/api-client-library/php", + "keywords": [ + "google" + ], + "support": { + "issues": "https://github.com/googleapis/google-api-php-client-services/issues", + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.370.0" + }, + "time": "2024-08-26T01:04:18+00:00" + }, + { + "name": "google/auth", + "version": "v1.42.0", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-auth-library-php.git", + "reference": "0c25599a91530b5847f129b271c536f75a7563f5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/0c25599a91530b5847f129b271c536f75a7563f5", + "reference": "0c25599a91530b5847f129b271c536f75a7563f5", + "shasum": "" + }, + "require": { + "firebase/php-jwt": "^6.0", + "guzzlehttp/guzzle": "^7.4.5", + "guzzlehttp/psr7": "^2.4.5", + "php": "^8.0", + "psr/cache": "^2.0||^3.0", + "psr/http-message": "^1.1||^2.0" + }, + "require-dev": { + "guzzlehttp/promises": "^2.0", + "kelvinmo/simplejwt": "0.7.1", + "phpseclib/phpseclib": "^3.0.35", + "phpspec/prophecy-phpunit": "^2.1", + "phpunit/phpunit": "^9.6", + "sebastian/comparator": ">=1.2.3", + "squizlabs/php_codesniffer": "^3.5", + "symfony/process": "^6.0||^7.0", + "webmozart/assert": "^1.11" + }, + "suggest": { + "phpseclib/phpseclib": "May be used in place of OpenSSL for signing strings or for token management. Please require version ^2." + }, + "type": "library", + "autoload": { + "psr-4": { + "Google\\Auth\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Google Auth Library for PHP", + "homepage": "http://github.com/google/google-auth-library-php", + "keywords": [ + "Authentication", + "google", + "oauth2" + ], + "support": { + "docs": "https://googleapis.github.io/google-auth-library-php/main/", + "issues": "https://github.com/googleapis/google-auth-library-php/issues", + "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.42.0" + }, + "time": "2024-08-26T18:33:48+00:00" + }, { "name": "grasmash/expander", "version": "3.0.0", @@ -10398,6 +10571,116 @@ }, "time": "2021-09-22T16:57:06+00:00" }, + { + "name": "phpseclib/phpseclib", + "version": "3.0.41", + "source": { + "type": "git", + "url": "https://github.com/phpseclib/phpseclib.git", + "reference": "621c73f7dcb310b61de34d1da4c4204e8ace6ceb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/621c73f7dcb310b61de34d1da4c4204e8ace6ceb", + "reference": "621c73f7dcb310b61de34d1da4c4204e8ace6ceb", + "shasum": "" + }, + "require": { + "paragonie/constant_time_encoding": "^1|^2|^3", + "paragonie/random_compat": "^1.4|^2.0|^9.99.99", + "php": ">=5.6.1" + }, + "require-dev": { + "phpunit/phpunit": "*" + }, + "suggest": { + "ext-dom": "Install the DOM extension to load XML formatted public keys.", + "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.", + "ext-libsodium": "SSH2/SFTP can make use of some algorithms provided by the libsodium-php extension.", + "ext-mcrypt": "Install the Mcrypt extension in order to speed up a few other cryptographic operations.", + "ext-openssl": "Install the OpenSSL extension in order to speed up a wide variety of cryptographic operations." + }, + "type": "library", + "autoload": { + "files": [ + "phpseclib/bootstrap.php" + ], + "psr-4": { + "phpseclib3\\": "phpseclib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jim Wigginton", + "email": "terrafrost@php.net", + "role": "Lead Developer" + }, + { + "name": "Patrick Monnerat", + "email": "pm@datasphere.ch", + "role": "Developer" + }, + { + "name": "Andreas Fischer", + "email": "bantu@phpbb.com", + "role": "Developer" + }, + { + "name": "Hans-Jürgen Petrich", + "email": "petrich@tronic-media.com", + "role": "Developer" + }, + { + "name": "Graham Campbell", + "email": "graham@alt-three.com", + "role": "Developer" + } + ], + "description": "PHP Secure Communications Library - Pure-PHP implementations of RSA, AES, SSH2, SFTP, X.509 etc.", + "homepage": "http://phpseclib.sourceforge.net", + "keywords": [ + "BigInteger", + "aes", + "asn.1", + "asn1", + "blowfish", + "crypto", + "cryptography", + "encryption", + "rsa", + "security", + "sftp", + "signature", + "signing", + "ssh", + "twofish", + "x.509", + "x509" + ], + "support": { + "issues": "https://github.com/phpseclib/phpseclib/issues", + "source": "https://github.com/phpseclib/phpseclib/tree/3.0.41" + }, + "funding": [ + { + "url": "https://github.com/terrafrost", + "type": "github" + }, + { + "url": "https://www.patreon.com/phpseclib", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpseclib/phpseclib", + "type": "tidelift" + } + ], + "time": "2024-08-12T00:13:54+00:00" + }, { "name": "phrity/net-uri", "version": "1.3.0", From 14b875fc7a021023748bedc2848bff9c67cba11d Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 4 Sep 2024 09:58:27 +0300 Subject: [PATCH 05/57] UHF-8706: intermediate commit with working index-check and single item indexing --- .../helfi_google_api.info.yml | 1 + .../src/Drush/Commands/HelfiApiCommands.php | 108 +++++++++- .../helfi_google_api/src/HelfiGoogleApi.php | 91 +++++++-- .../src/JobIndexingService.php | 184 ++++++++++++++++-- 4 files changed, 357 insertions(+), 27 deletions(-) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml index 93d13261..ecd47da5 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml @@ -5,3 +5,4 @@ package: HELfi core_version_requirement: ^10 || ^11 dependencies: - 'publication_date:publication_date' + - redirect:redirect diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 012c0825..71de8754 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -5,6 +5,9 @@ namespace Drupal\helfi_google_api\Drush\Commands; use Drupal\helfi_google_api\JobIndexingService; +use Drupal\helfi_rekry_content\Entity\JobListing; +use Drupal\node\Entity\Node; +use Drupal\redirect\Entity\Redirect; use Drush\Attributes\Command; use Drush\Commands\AutowireTrait; use Drush\Commands\DrushCommands; @@ -22,9 +25,108 @@ public function __construct( parent::__construct(); } - #[Command(name: 'helfi:index-google')] - public function process() : int { - $this->jobIndexingService->indexUpdatedJobs(); + #[Command(name: 'helfi:single-index-google')] + public function indexSingleItem( + int $entity_id, + string $langcode = 'fi', + ) : int { + $job = Node::load($entity_id); + if (!$job instanceof JobListing) { + $this->io()->writeln('Entity not found or not instance of JobListing'); + return DrushCommands::EXIT_FAILURE; + } + + $job_alias = \Drupal::service('path_alias.manager')->getAliasByPath("/node/{$job->id()}", $langcode); + + // Check if entity has already been indexed. + $redirectIds = \Drupal::entityQuery('redirect') + ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") + ->condition('status_code', 301) + ->condition('language', $langcode) + ->accessCheck(FALSE) + ->execute(); + + $redirects = Redirect::loadMultiple($redirectIds); + + foreach ($redirects as $redirect) { + $source = $redirect->getSourceUrl(); + + if (str_contains($source, "$job_alias-")) { + $this->io()->writeln('Entity has temporary redirect which indicates + that indexing has already been requested.'); + return DrushCommands::EXIT_FAILURE; + } + } + + $now = strtotime('now'); + $temp_alias = "$job_alias-$now"; + $indexing_url = "{$job->toUrl()->setAbsolute()->toString()}-$now"; + + $redirect = Redirect::create([ + 'redirect_source' => ltrim($temp_alias, '/'), + 'redirect_redirect' => "internal:/node/{$job->id()}", + 'language' => $langcode, + 'status_code' => 301, + ]); + + try { + $redirect->save(); + } + catch (\Exception $e) { + $this->io()->writeln('Failed to create temporary redirect for the entity.'); + return DrushCommands::EXIT_FAILURE; + } + + $this->jobIndexingService->indexItem($indexing_url); + return DrushCommands::EXIT_SUCCESS; + } + + #[Command(name: 'helfi:google-index-status-check')] + public function checkItemIndexStatus(int $entity_id, $langcode = 'fi') { + $job = Node::load($entity_id); + if (!$job instanceof JobListing) { + $this->io()->writeln('Entity not found or not instance of JobListing'); + return DrushCommands::EXIT_FAILURE; + } + + $language = \Drupal::languageManager()->getLanguage($langcode); + $baseUrl = \Drupal::urlGenerator()->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); + $job_alias = \Drupal::service('path_alias.manager')->getAliasByPath("/node/{$job->id()}", $langcode); + + // Check if entity has already been indexed. + $redirectIds = \Drupal::entityQuery('redirect') + ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") + ->condition('status_code', 301) + ->condition('language', $langcode) + ->accessCheck(FALSE) + ->execute(); + + // Get the indexed redirect. + $redirects = Redirect::loadMultiple($redirectIds); + foreach ($redirects as $redirect) { + $source = $redirect->getSourceUrl(); + + if (str_contains($source, "$job_alias-")) { + $correct_redirect = $redirect; + break; + } + } + + if (!$correct_redirect) { + $this->io()->writeln('Redirect request has not been sent.'); + return DrushCommands::EXIT_FAILURE; + } + + $url_to_check = $baseUrl . $redirect->getSourceUrl(); + try { + $response = $this->jobIndexingService->checkItemIndexStatus($url_to_check); + } + catch (\Exception $e) { + $this->io()->writeln('Something went wrong: ' . $e->getMessage()); + return DrushCommands::EXIT_FAILURE; + } + + $this->io()->writeln($response); return DrushCommands::EXIT_SUCCESS; } diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index af177fc1..d02a3cab 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -5,30 +5,95 @@ namespace Drupal\helfi_google_api; use Google\Client as GoogleClient; -use Google\Service\Indexing\UrlNotification; use Google\Service\Indexing; +use GuzzleHttp\Psr7\Request; class HelfiGoogleApi { - const END_POINT = 'https://indexing.googleapis.com/v3/urlNotifications:publish'; + const PUBLISH_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications:publish'; + + const METADATA_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications/metadata'; + + const BATCH_ENDPOINT = 'https://indexing.googleapis.com/batch'; const SCOPES = ['https://www.googleapis.com/auth/indexing']; + const UPDATE = 'URL_UPDATED'; + + // todo Could be URL_REMOVED + const DELETE = 'URL_DELETED'; + + private Indexing $indexingService; + public function __construct( - private readonly GoogleClient $apiClient ) { - // $indexing = new Indexing(); - // $apiClient->setApplicationName('Helfi_Rekry'); - //$indexing = new UrlNotification(); - // $apiClient->setDeveloperKey(); - // $apiClient->setAuthConfig(); - // $this->apiClient->setUseBatch(TRUE); - // $batch = $this->apiClient->execute(); - - // $apiClient->setScopes(self::SCOPES); } - public function indexJobs(array $jobs, string $type): void { + /** + * Send update request to indexing api, no more than 100 items at a time. + * + * @param array $jobs + * Array of job_listing nodes to be indexed or removed from index. + * + * @return void + */ + public function indexBatch(array $urls, bool $update): void { + + $this->initializeApi(); + $batch = $this->indexingService->createBatch(); + $operation = $update ? self::UPDATE : self::DELETE; + + foreach ($urls as $url) { + $content = [ + 'type' => $operation, + 'url' => $url, + ]; + + $request = new Request( + method: 'POST', + uri: self::PUBLISH_ENDPOINT, + headers: ['Content-Type' => 'multipart/mixed'], + body: json_encode($content) + ); + + $batch->add($request); + } + + die('LETS NOT SEND STUFF JUST YET'); + // $batch->execute(); + } + + public function checkIndexingStatus($url): string { + $base_url = self::METADATA_ENDPOINT; + $queryParameter = '?url=' . urlencode($url); + $the_url = $base_url . $queryParameter; + + $this->initializeApi(FALSE); + $client = $this->indexingService->getClient()->authorize(); + $response = $client->get($the_url); + + return $response->getBody()->getContents(); + } + + private function initializeApi($batch = TRUE) { + // @todo Get correct key. + $key = 'CHANGE THIS'; + $config = \Drupal::configFactory()->get('helfi_google_api.settings'); + $key = $config->get('indexing_api_key') ?: ''; + + if (!$key) { + throw new \Exception('Api key not configured. Unable to proceed.'); + } + + $client = new GoogleClient(); + $client->setApplicationName('Helfi_Rekry'); + // $client->setDeveloperKey($key); + $client->setAuthConfig(json_decode($key, TRUE)); + + $client->addScope(self::SCOPES); + $client->setUseBatch($batch); + + $this->indexingService = new Indexing($client); } } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 30cb512c..124ea225 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -5,37 +5,199 @@ namespace Drupal\helfi_google_api; use Drupal\Core\Entity\EntityTypeManagerInterface; -use Drupal\helfi_google_api\HelfiGoogleApi; +use Drupal\path_alias\AliasManagerInterface; +use Drupal\redirect\Entity\Redirect; +use GuzzleHttp\Exception\GuzzleException; class JobIndexingService { public function __construct( private readonly EntityTypeManagerInterface $entityTypeManager, private readonly HelfiGoogleApi $helfiGoogleApi, + private readonly AliasManagerInterface $aliasManager, ) { } /** - * Update any jobs that were either published or unpublished today. + * Send single item to google for indexing. + * + * @param string $url + * Url to index. * * @return void */ - public function indexUpdatedJobs(): void { + public function indexItem(string $url) { + $this->helfiGoogleApi->indexBatch([$url], TRUE); + } + + public function deindexItem(string $url) { + $this->helfiGoogleApi->indexBatch([$url], FALSE); + } + + public function checkItemIndexStatus(string $url): string { + try { + return $this->helfiGoogleApi->checkIndexingStatus($url); + } + catch (GuzzleException $e) { + throw new \Exception($e->getMessage(), $e->getCode()); + } + } + + /** + * Update any jobs that were published today. + * + * @return void + */ + public function indexJobs(): void { $storage = $this->entityTypeManager - ->getStorage('job_listing'); - $startOfToday = strtotime('today 00:05'); - $startOfYesterday = strtotime('yesterday 00:05'); + ->getStorage('node'); + $startOfToday = strtotime('today 00:00:00'); + $endOfToday = strtotime('today 23:59:59'); + + // Etsi nodet joiden publish_on on tänään + // TAI ei oo tänään mutta on luotu tänään - $this->helfiGoogleApi->indexJobs(); - $publishedJobIds = $storage->getQuery() - ->condition('publish_on', $startOfToday, '>=') + $jobIds = $storage->getQuery() + // ->condition('publish_on', $startOfToday, '>=') + // ->condition('publish_on', $endOfToday, '<=') ->condition('status', 1) + ->condition('type', 'job_listing') + ->latestRevision() ->accessCheck(FALSE) ->execute(); - $publishedJobs = $storage->loadMultiple($publishedJobIds); - $this->helfiGoogleApi->indexJobs($publishedJobs, 'update'); + if (!$jobIds) { + return; + } + + $jobs = $storage->loadMultiple($jobIds); + $urls = $this->prepareUrls($jobs); + + $this->helfiGoogleApi->indexJobsBatch($urls); + } + + public function deleteIndexedJobs() { + + $storage = $this->entityTypeManager + ->getStorage('node'); + $startOfToday = strtotime('today 00:00:00'); + $endOfToday = strtotime('today 23:59:59'); + + // Etsi nodet joiden unpublish on on tänään + $jobIds = $storage->getQuery() + ->condition('unpublish_on', $startOfToday, '>=') + ->condition('unpublish_on', $endOfToday, '<=') + ->condition('status', 0) + ->condition('type', 'job_listing') + ->latestRevision() + ->accessCheck(FALSE) + ->execute(); + + if (!$jobIds) { + return; + } + + $jobs = $storage->loadMultiple($jobIds); + + // hae redirectit + $redirect = $this->getRedirects($jobs); + + // kaiva se urli sieltä + // lähetä googlelle + // poista redirect + + // $this->helfiGoogleApi->indexJobsBatch($urls); + // $urls = $this->getUrls($jobs); + + // Redirect::load() + + } + + // TODO get urls function should not create redirects + private function prepareUrls(array $jobs, array $langcodes = ['fi', 'en', 'sv']): array { + $result = []; + + $now = strtotime('now'); + + foreach ($jobs as $job) { + + foreach($langcodes as $langcode) { + if ($job->hasTranslation($langcode)) { + $job = $job->getTranslation($langcode); + $alias = $this->aliasManager->getAliasByPath("/node/{$job->id()}", $langcode); + $temp_alias = "{$alias}-{$now}"; + + $redirect = Redirect::create([ + 'redirect_source' => ltrim($temp_alias, '/'), + 'redirect_redirect' => "internal:/node/{$job->id()}", + 'language' => $langcode, + 'status_code' => 301, + 'title' => $job->getTitle(), + ]); + + // $redirect->setRedirect( "/node/{$job->id()}", [], ['redirect_title' => $job->getTitle()]); + $save_status = $redirect->save(); + + if ($save_status === SAVED_NEW) { + $index_url = "{$job->toUrl()->setAbsolute(TRUE)->toString()}-{$now}"; + $result[] = $index_url; + } + + } + } + } + + return $result; + } + + /** + * Delete a temporary redirect. + * + * @param array $jobs + * Array of nodes. + * + * @return array + */ + private function deleteRedirect(array $jobs): array { + $result = []; + $langcodes = ['fi', 'en', 'sv']; + + foreach ($jobs as $job) { + foreach($langcodes as $langcode) { + if ($job->hasTranslation($langcode)) { + $job = $job->getTranslation($langcode); + + $redirectIds = \Drupal::entityQuery('redirect') + ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") + ->condition('status_code', 301) + ->condition('language', $langcode) + ->execute(); + + $redirects = Redirect::loadMultiple($redirectIds); + /** @var Redirect $redirect */ + foreach ($redirects as $redirect) { + $alias = $this->aliasManager->getAliasByPath("/node/{$job->id()}", $langcode); + $source = $redirect->getSourceUrl(); + if (!str_contains($source, "$alias-")) { + continue; + } + + $result[] = $source; + try { + $redirect->delete(); + } + catch(\Exception $e) { + // Add logging. + } + + } + } + } + + return $result; + } + return $result; } } From 784144dc13af9757cd87c2827442469c5e78f051 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 4 Sep 2024 10:00:12 +0300 Subject: [PATCH 06/57] UHF-8706: enable module, added module settings config --- conf/cmi/core.extension.yml | 1 + conf/cmi/helfi_google_api.settings.yml | 1 + 2 files changed, 2 insertions(+) create mode 100644 conf/cmi/helfi_google_api.settings.yml diff --git a/conf/cmi/core.extension.yml b/conf/cmi/core.extension.yml index 13281915..122b5518 100644 --- a/conf/cmi/core.extension.yml +++ b/conf/cmi/core.extension.yml @@ -38,6 +38,7 @@ module: flysystem: 0 focal_point: 0 gin_toolbar: 0 + google_index_api: 0 hdbt_admin_tools: 0 health_check: 0 helfi_api_base: 0 diff --git a/conf/cmi/helfi_google_api.settings.yml b/conf/cmi/helfi_google_api.settings.yml new file mode 100644 index 00000000..8407ee2e --- /dev/null +++ b/conf/cmi/helfi_google_api.settings.yml @@ -0,0 +1 @@ +indexing_api_key: '' From ac4b18075babf607df1c0db5f92f7290149d1970 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 4 Sep 2024 10:20:11 +0300 Subject: [PATCH 07/57] UHF-8706: remove the helfi_google_api contrib module from core.extension --- conf/cmi/core.extension.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/cmi/core.extension.yml b/conf/cmi/core.extension.yml index 122b5518..13281915 100644 --- a/conf/cmi/core.extension.yml +++ b/conf/cmi/core.extension.yml @@ -38,7 +38,6 @@ module: flysystem: 0 focal_point: 0 gin_toolbar: 0 - google_index_api: 0 hdbt_admin_tools: 0 health_check: 0 helfi_api_base: 0 From 54dbbac4d9c62f594f9ff3e621ff8afb48e55038 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 5 Sep 2024 10:06:05 +0300 Subject: [PATCH 08/57] UHF-8706: add logger --- .../helfi_google_api.services.yml | 7 ++- .../JobPublishStateSubscriber.php | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index 247d728e..80207837 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -9,5 +9,10 @@ services: Drupal\helfi_google_api\HelfiGoogleApi: ~ - Drupal\helfi_google_api\JobIndexingService: ~ + Drupal\helfi_google_api\JobIndexingService: + class: Drupal\helfi_google_api\JobIndexingService + autowire: true + arguments: + $logger: '@logger.channel.helfi_google_api' + Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: ~ diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php new file mode 100644 index 00000000..51f0b890 --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -0,0 +1,43 @@ + 'prepareIndices', + ]; + } + + /** + * Method to prepare index. + * + * @param \Drupal\elasticsearch_connector\Event\PrepareIndexEvent $event + * The PrepareIndex event. + */ + public function prepareIndices(PrepareIndexEvent $event) { + $indexName = $event->getIndexName(); + $finnishIndices = [ + 'job_listings', + ]; + if (in_array($indexName, $finnishIndices)) { + /** @var array $indexConfig */ + $indexConfig = $event->getIndexConfig(); + $indexConfig['body']['settings']['analysis']['analyzer']['default']['type'] = 'finnish'; + $event->setIndexConfig($indexConfig); + } + } + +} From f5f2543aa599c26a606f37b7997edeb98c510bea Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 5 Sep 2024 10:06:54 +0300 Subject: [PATCH 09/57] UHF-8706: eventsubscriber to handle joblistings published by scheduler --- .../JobPublishStateSubscriber.php | 98 ++++++++++++++++--- 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 51f0b890..cde622c8 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -4,40 +4,106 @@ namespace Drupal\helfi_google_api\EventSubscriber; -use Drupal\elasticsearch_connector\Event\PrepareIndexEvent; +use Drupal\helfi_google_api\JobIndexingService; +use Drupal\helfi_rekry_content\Entity\JobListing; +use Drupal\redirect\Entity\Redirect; +use Drupal\scheduler\SchedulerEvent; +use Drupal\scheduler\SchedulerEvents; +use Drush\Commands\DrushCommands; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** * {@inheritdoc} */ -class JobPublishSubscriber implements EventSubscriberInterface { +class JobPublishStateSubscriber implements EventSubscriberInterface { + + public function __construct( + private readonly JobIndexingService $jobIndexingService, + ) { + } + /** * {@inheritdoc} */ public static function getSubscribedEvents() { return [ - PrepareIndexEvent::PREPARE_INDEX => 'prepareIndices', + SchedulerEvents::PUBLISH => 'sendIndexingRequest', + SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', + SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', ]; } + public function sendIndexingRequest(SchedulerEvent $event) { + $entity = $event->getNode(); + if (!$entity instanceof JobListing) { + return; + } + + $langcodes = ['fi', 'en', 'sv']; + $results = []; + + foreach($langcodes as $langcode) { + try { + $hasRedirect = $this->jobIndexingService->temporaryRedirectExists($entity, $langcode); + if ($hasRedirect) { + // log, continue. + continue; + } + } + catch (\Exception $e) { + // Log. + continue; + } + + try { + $indexing_url = $this->jobIndexingService->createTemporaryRedirectUrl($entity, $langcode); + } + catch (\Exception $e) { + // cannot create url, log + continue; + } + + $results[] = $indexing_url; + } + + $result = $this->jobIndexingService->indexItems($results); + + if ($result['errors']) { + // Some of the urls failed + // log + } + + } + /** - * Method to prepare index. + * Send deindexing request to google. * - * @param \Drupal\elasticsearch_connector\Event\PrepareIndexEvent $event - * The PrepareIndex event. + * @param SchedulerEvent $event + * @return void */ - public function prepareIndices(PrepareIndexEvent $event) { - $indexName = $event->getIndexName(); - $finnishIndices = [ - 'job_listings', - ]; - if (in_array($indexName, $finnishIndices)) { - /** @var array $indexConfig */ - $indexConfig = $event->getIndexConfig(); - $indexConfig['body']['settings']['analysis']['analyzer']['default']['type'] = 'finnish'; - $event->setIndexConfig($indexConfig); + public function sendDeindexingRequest(SchedulerEvent $event) { + $entity = $event->getNode(); + $langcode = $entity->language()->getId(); + if (!$entity instanceof JobListing) { + return; } + + $redirect = $this->jobIndexingService->getExistingTemporaryRedirect($entity, $langcode); + if (!$redirect) { + // Log, the item seems not to be indexed. + return; + } + + $base_url = \Drupal::urlGenerator()->generateFromRoute( + '', + [], + ['absolute' => TRUE,'language' => $entity->language()] + ); + + $url_to_deindex = $base_url . $redirect->getSourceUrl(); + + $this->jobIndexingService->deindexItems([$url_to_deindex]); } } From 315dc6cb3887cff88dd43b7627d1b13859497280 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 5 Sep 2024 10:09:29 +0300 Subject: [PATCH 10/57] UHF-8706: refactored by moving the logic to service, added docblocks --- .../helfi_google_api/src/HelfiGoogleApi.php | 94 +++++++-- .../src/JobIndexingService.php | 179 +++++++++++++++++- 2 files changed, 246 insertions(+), 27 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index d02a3cab..7137e27c 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -4,41 +4,72 @@ namespace Drupal\helfi_google_api; +use Drupal\Core\Config\ConfigFactoryInterface; use Google\Client as GoogleClient; +use Google\Service\Exception; use Google\Service\Indexing; use GuzzleHttp\Psr7\Request; +/** + * A wrapper for Google indexing library. + */ class HelfiGoogleApi { + /** + * Endpoint for sending update and delete requests. + */ const PUBLISH_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications:publish'; + /** + * Endpoint for requesting url indexing status. + */ const METADATA_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications/metadata'; - const BATCH_ENDPOINT = 'https://indexing.googleapis.com/batch'; - + /** + * Api scopes. + */ const SCOPES = ['https://www.googleapis.com/auth/indexing']; + /** + * Request type for update-request (indexing). + */ const UPDATE = 'URL_UPDATED'; - // todo Could be URL_REMOVED + /** + * Request type for delete request (deindexing). + */ const DELETE = 'URL_DELETED'; + /** + * Google indexing service. + * + * @var Google\Service\Indexing + */ private Indexing $indexingService; + /** + * The constructor. + * + * @param Drupal\Core\Config\ConfigFactoryInterface $configFactory + * The config factory. + */ public function __construct( + private readonly ConfigFactoryInterface $configFactory, ) { } /** - * Send update request to indexing api, no more than 100 items at a time. + * Send indexing or deindexing request for urls. * - * @param array $jobs - * Array of job_listing nodes to be indexed or removed from index. + * @param array $urls + * Array of urls to index or deindex. + * @param bool $update + * TRUE to index the urls, FALSE for deindexing. * - * @return void + * @return array + * Array which consists of the amount of items sent and array of errors. */ - public function indexBatch(array $urls, bool $update): void { - + public function indexBatch(array $urls, bool $update): array { $this->initializeApi(); $batch = $this->indexingService->createBatch(); $operation = $update ? self::UPDATE : self::DELETE; @@ -59,26 +90,56 @@ public function indexBatch(array $urls, bool $update): void { $batch->add($request); } - die('LETS NOT SEND STUFF JUST YET'); - // $batch->execute(); + $responses = $batch->execute(); + + $errors = []; + + foreach ($responses as $key => $response) { + if ($response instanceof Exception) { + $errors[] = "$key: {$response->getMessage()}"; + } + } + + return [ + 'count' => count($urls), + 'errors' => $errors, + ]; } + /** + * Request url indexing status. + * + * Returns the dates of last update and delete requests. + * + * @param string $url + * The url which indexing status you want to request. + * + * @return string + * The response as a string. + */ public function checkIndexingStatus($url): string { + $this->initializeApi(FALSE); + $base_url = self::METADATA_ENDPOINT; $queryParameter = '?url=' . urlencode($url); $the_url = $base_url . $queryParameter; - $this->initializeApi(FALSE); $client = $this->indexingService->getClient()->authorize(); $response = $client->get($the_url); return $response->getBody()->getContents(); } - private function initializeApi($batch = TRUE) { - // @todo Get correct key. - $key = 'CHANGE THIS'; - $config = \Drupal::configFactory()->get('helfi_google_api.settings'); + /** + * Set up the client. + * + * @param bool $batch + * Set the client on batch mode. + * + * @return void + */ + private function initializeApi(bool $batch = TRUE): void { + $config = $this->configFactory->get('helfi_google_api.settings'); $key = $config->get('indexing_api_key') ?: ''; if (!$key) { @@ -87,7 +148,6 @@ private function initializeApi($batch = TRUE) { $client = new GoogleClient(); $client->setApplicationName('Helfi_Rekry'); - // $client->setDeveloperKey($key); $client->setAuthConfig(json_decode($key, TRUE)); $client->addScope(self::SCOPES); diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 124ea225..6c7a0882 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -4,34 +4,194 @@ namespace Drupal\helfi_google_api; +use Drupal\Core\DependencyInjection\AutowireTrait; use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\Routing\UrlGeneratorInterface; +use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\path_alias\AliasManagerInterface; use Drupal\redirect\Entity\Redirect; use GuzzleHttp\Exception\GuzzleException; +use Psr\Log\LoggerInterface; +/** + * Send indexing requests to Google indexing api. + */ class JobIndexingService { + use AutowireTrait; public function __construct( - private readonly EntityTypeManagerInterface $entityTypeManager, private readonly HelfiGoogleApi $helfiGoogleApi, + private readonly EntityTypeManagerInterface $entityTypeManager, private readonly AliasManagerInterface $aliasManager, + private readonly UrlGeneratorInterface $urlGenerator, + private readonly LoggerInterface $logger, ) { } /** - * Send single item to google for indexing. + * Send urls to google for indexing. * - * @param string $url - * Url to index. + * @param array $urls + * Array of urls to index. * - * @return void + * @return array + * Array of containing total count indexed and errors. + */ + public function indexItems(array $urls): array { + return $this->helfiGoogleApi->indexBatch($urls, TRUE); + } + + /** + * Send indexing request to google. + * + * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * Entity which indexing should be requested. + * + * @return array + * Array of containing total count indexed and errors. + */ + public function indexEntity(JobListing $entity): array { + $langcode = $entity->language()->getId(); + $results = []; + + // If the actual url is deindexed, it can't be reindexed again. + // If entity has a temporary redirect, it most likely has already been indexed already. + $hasRedirect = $this->temporaryRedirectExists($entity, $langcode); + if ($hasRedirect) { + throw new \Exception('Already indexed.'); + } + + // Create temporary redirect for the entity. + $indexing_url = $this->createTemporaryRedirectUrl($entity, $langcode); + + $result = $this->indexItems([$indexing_url]); + + if ($result['errors']) { + // Some of the urls failed + // log + } + + return $result; + } + + + /** + * Send urls to Google for deindexing. + * + * @param array $urls + * Urls to remove from index. + * + * @return array + * + */ + public function deindexItems(array $urls): array { + return $this->helfiGoogleApi->indexBatch($urls, FALSE); + } + + /** + * Handle entity indexing request + * + * @param JobListing $entity + * @return array + * @throws \Exception + */ + public function deindexEntity(JobListing $entity): array { + $language = $entity->language(); + $redirect = $this->getExistingTemporaryRedirect($entity, $language->getId()); + if (!$redirect) { + // Log, the item seems not to be indexed. + // Return. + throw new \Exception(); + } + + $base_url = $this->urlGenerator->generateFromRoute( + '', + [], + ['absolute' => TRUE,'language' => $language] + ); + + $url_to_deindex = $base_url . $redirect->getSourceUrl(); + + return $this->deindexItems([$url_to_deindex]); + } + + private function temporaryRedirectExists(JobListing $entity, string $langcode): bool { + $job_alias = $this->getEntityAlias($entity, $langcode); + // Check if entity has already been indexed. + + $query = $this->entityTypeManager->getStorage('redirect') + ->getQuery(); + + $redirectIds = $query->condition('redirect_redirect__uri', "internal:/node/{$entity->id()}") + ->condition('status_code', 301) + ->condition('language', $langcode) + ->accessCheck(FALSE) + ->execute(); + $redirects = Redirect::loadMultiple($redirectIds); + + foreach ($redirects as $redirect) { + $source = $redirect->getSourceUrl(); + + if (str_contains($source, "$job_alias-")) { + return TRUE; + } + } + + return FALSE; + } + + /** + * Create a redirect for the indexing request. + * + * @param JobListing $entity + * The entity to index. + * @param string $langcode + * + * + * @return string + * Absolute url that can be send for indexing. */ - public function indexItem(string $url) { - $this->helfiGoogleApi->indexBatch([$url], TRUE); + private function createTemporaryRedirectUrl(JobListing $entity, string $langcode): string { + $alias = $this->getEntityAlias($entity, $langcode); + $now = strtotime('now'); + $temp_alias = "$alias-$now"; + $indexing_url = "{$entity->toUrl()->setAbsolute()->toString()}-$now"; + + Redirect::create([ + 'redirect_source' => ltrim($temp_alias, '/'), + 'redirect_redirect' => "internal:/node/{$entity->id()}", + 'language' => $langcode, + 'status_code' => 301, + ])->save(); + + return $indexing_url; + } + + private function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { + $job_alias = $this->getEntityAlias($entity, $langcode); + + $query = $this->entityTypeManager->getStorage('redirect') + ->getQuery(); + + $redirectIds = $query->condition('redirect_redirect__uri', "internal:/node/{$entity->id()}") + ->condition('status_code', 301) + ->condition('language', $langcode) + ->accessCheck(FALSE) + ->execute(); + $redirects = Redirect::loadMultiple($redirectIds); + + foreach ($redirects as $redirect) { + $source = $redirect->getSourceUrl(); + + if (str_contains($source, "$job_alias-")) { + return $redirect; + } + } + return NULL; } - public function deindexItem(string $url) { - $this->helfiGoogleApi->indexBatch([$url], FALSE); + private function getEntityAlias(JobListing $entity, string $langcode): string { + return $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $langcode); } public function checkItemIndexStatus(string $url): string { @@ -113,7 +273,6 @@ public function deleteIndexedJobs() { } - // TODO get urls function should not create redirects private function prepareUrls(array $jobs, array $langcodes = ['fi', 'en', 'sv']): array { $result = []; From a0850bd8e4d6b50db93b0512293f2a0e18dcef11 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 5 Sep 2024 12:02:50 +0300 Subject: [PATCH 11/57] UHF-8706: added more docblocks, added url check code to the service --- .../src/Drush/Commands/HelfiApiCommands.php | 141 ++++++++++-------- .../src/JobIndexingService.php | 125 ++++++++++++++-- 2 files changed, 188 insertions(+), 78 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 71de8754..9e4b004e 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -4,13 +4,18 @@ namespace Drupal\helfi_google_api\Drush\Commands; +use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\Language\LanguageManagerInterface; +use Drupal\Core\Routing\UrlGeneratorInterface; use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\node\Entity\Node; +use Drupal\path_alias\AliasManagerInterface; use Drupal\redirect\Entity\Redirect; use Drush\Attributes\Command; use Drush\Commands\AutowireTrait; use Drush\Commands\DrushCommands; +use Symfony\Component\HttpFoundation\RedirectResponse; /** * A Drush command file. @@ -21,108 +26,114 @@ final class HelfiApiCommands extends DrushCommands { public function __construct( private readonly JobIndexingService $jobIndexingService, + private readonly EntityTypeManagerInterface $entityTypeManager, + private readonly LanguageManagerInterface $languageManager, + private readonly AliasManagerInterface $aliasManager, + private readonly UrlGeneratorInterface $urlGenerator, ) { parent::__construct(); } + /** + * Index single job listing. + * + * @param int $entity_id + * The entity id. + * @param string $langcode + * The language code. + * + * @return int + * The exit code. + */ #[Command(name: 'helfi:single-index-google')] public function indexSingleItem( int $entity_id, string $langcode = 'fi', ) : int { - $job = Node::load($entity_id); - if (!$job instanceof JobListing) { + $entity = Node::load($entity_id); + + if (!$entity instanceof JobListing) { $this->io()->writeln('Entity not found or not instance of JobListing'); return DrushCommands::EXIT_FAILURE; } - $job_alias = \Drupal::service('path_alias.manager')->getAliasByPath("/node/{$job->id()}", $langcode); - - // Check if entity has already been indexed. - $redirectIds = \Drupal::entityQuery('redirect') - ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") - ->condition('status_code', 301) - ->condition('language', $langcode) - ->accessCheck(FALSE) - ->execute(); - - $redirects = Redirect::loadMultiple($redirectIds); - - foreach ($redirects as $redirect) { - $source = $redirect->getSourceUrl(); + if (!$entity->hasTranslation($langcode)) { + $this->io()->writeln('Translation does not exist.'); + return DrushCommands::EXIT_FAILURE; + } + $entity = $entity->getTranslation($langcode); - if (str_contains($source, "$job_alias-")) { - $this->io()->writeln('Entity has temporary redirect which indicates - that indexing has already been requested.'); - return DrushCommands::EXIT_FAILURE; - } + try { + $response = $this->jobIndexingService->indexEntity($entity); + } + catch (\Exception $e) { + $this->io()->error($e->getMessage()); + return DrushCommands::EXIT_FAILURE; } - $now = strtotime('now'); - $temp_alias = "$job_alias-$now"; - $indexing_url = "{$job->toUrl()->setAbsolute()->toString()}-$now"; + if ($response['errors']) { + $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response['errors'])); + return DrushCommands::EXIT_FAILURE_WITH_CLARITY; + } - $redirect = Redirect::create([ - 'redirect_source' => ltrim($temp_alias, '/'), - 'redirect_redirect' => "internal:/node/{$job->id()}", - 'language' => $langcode, - 'status_code' => 301, - ]); + $this->io()->writeln('Url indexed succesfully.'); + return DrushCommands::EXIT_SUCCESS; + } + /** + * Request url indexing status from Google api. + * + * @param int $url + * The url to check. + * + * @return int + * The exit code. + */ + #[Command(name: 'helfi:google-url-index-status')] + public function checkUrlIndexStatus(string $url) { try { - $redirect->save(); + $response = $this->jobIndexingService->checkItemIndexStatus($url); } catch (\Exception $e) { - $this->io()->writeln('Failed to create temporary redirect for the entity.'); + $this->io()->writeln($e->getMessage()); return DrushCommands::EXIT_FAILURE; } - $this->jobIndexingService->indexItem($indexing_url); + $this->io()->writeln($response); return DrushCommands::EXIT_SUCCESS; } - #[Command(name: 'helfi:google-index-status-check')] - public function checkItemIndexStatus(int $entity_id, $langcode = 'fi') { - $job = Node::load($entity_id); - if (!$job instanceof JobListing) { + /** + * Check entity indexing status. + * + * @param int $entity_id + * The entity id. + * @param string $langcode + * The language code. + * + * @return int + * The exit code. + */ + #[Command(name: 'helfi:google-entity-index-status')] + public function checkEntityIndexStatus(int $entity_id, $langcode = 'fi') { + $entity = Node::load($entity_id); + + if (!$entity instanceof JobListing) { $this->io()->writeln('Entity not found or not instance of JobListing'); return DrushCommands::EXIT_FAILURE; } - $language = \Drupal::languageManager()->getLanguage($langcode); - $baseUrl = \Drupal::urlGenerator()->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); - $job_alias = \Drupal::service('path_alias.manager')->getAliasByPath("/node/{$job->id()}", $langcode); - - // Check if entity has already been indexed. - $redirectIds = \Drupal::entityQuery('redirect') - ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") - ->condition('status_code', 301) - ->condition('language', $langcode) - ->accessCheck(FALSE) - ->execute(); - - // Get the indexed redirect. - $redirects = Redirect::loadMultiple($redirectIds); - foreach ($redirects as $redirect) { - $source = $redirect->getSourceUrl(); - - if (str_contains($source, "$job_alias-")) { - $correct_redirect = $redirect; - break; - } - } - - if (!$correct_redirect) { - $this->io()->writeln('Redirect request has not been sent.'); + if (!$entity->hasTranslation($langcode)) { + $this->io()->writeln('Translation does not exist.'); return DrushCommands::EXIT_FAILURE; } + $entity = $entity->getTranslation($langcode); - $url_to_check = $baseUrl . $redirect->getSourceUrl(); try { - $response = $this->jobIndexingService->checkItemIndexStatus($url_to_check); + $response = $this->jobIndexingService->checkEntityIndexStatus($entity); } catch (\Exception $e) { - $this->io()->writeln('Something went wrong: ' . $e->getMessage()); + $this->io()->writeln($e->getMessage()); return DrushCommands::EXIT_FAILURE; } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 6c7a0882..dc6f0904 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -82,7 +82,7 @@ public function indexEntity(JobListing $entity): array { * Urls to remove from index. * * @return array - * + * Array: 'count': int, 'errors': array. */ public function deindexItems(array $urls): array { return $this->helfiGoogleApi->indexBatch($urls, FALSE); @@ -92,8 +92,10 @@ public function deindexItems(array $urls): array { * Handle entity indexing request * * @param JobListing $entity + * Entity to request deindexing for. + * * @return array - * @throws \Exception + * Array: 'count': int, 'errors': array */ public function deindexEntity(JobListing $entity): array { $language = $entity->language(); @@ -115,9 +117,88 @@ public function deindexEntity(JobListing $entity): array { return $this->deindexItems([$url_to_deindex]); } + /** + * Check url indexing status. + * + * @param string $url + * An url to check. + * + * @return string + * Status as a string. + */ + public function checkItemIndexStatus(string $url): string { + try { + return $this->helfiGoogleApi->checkIndexingStatus($url); + } + catch (GuzzleException $e) { + throw new \Exception($e->getMessage(), $e->getCode()); + } + } + + /** + * If entity seems to be indexed, send a status query. + * + * @param JobListing $entity + * @return array + */ + public function checkEntityIndexStatus(JobListing $entity): string { + $language = $entity->language(); + + $baseUrl = $this->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); + $job_alias = $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $language->getId()); + + $query = $this->entityTypeManager->getStorage('redirect')->getQuery(); + $redirectIds = $query->condition('redirect_redirect__uri', "internal:/node/{$entity->id()}") + ->condition('status_code', 301) + ->condition('language', $language->getId()) + ->accessCheck(FALSE) + ->execute(); + + // Get the indexed redirect. + $redirects = Redirect::loadMultiple($redirectIds); + foreach ($redirects as $redirect) { + $source = $redirect->getSourceUrl(); + + if (str_contains($source, "$job_alias-")) { + $correct_redirect = $redirect; + break; + } + } + + if (!$correct_redirect) { + // HAS NOT BEEN SENT AFAIK + } + + $url_to_check = $baseUrl . $correct_redirect->getSourceUrl(); + try { + $response = $this->helfiGoogleApi->checkIndexingStatus($url_to_check); + } + catch (\Exception $e) { + // REQUEST FAILED + + } + + return $response; + + } + + /** + * Does the entity have a temporary redirect. + * + * Temporary redirect is created for all entities before requesting indexing. + * Once delete request is sent, the url cannot be indexed again using the api. + * Hence we should not use the original url. + * + * @param JobListing $entity + * The entity to check. + * @param string $langcode + * The language code. + * + * @return bool + * Has temporary redirect. + */ private function temporaryRedirectExists(JobListing $entity, string $langcode): bool { $job_alias = $this->getEntityAlias($entity, $langcode); - // Check if entity has already been indexed. $query = $this->entityTypeManager->getStorage('redirect') ->getQuery(); @@ -143,13 +224,17 @@ private function temporaryRedirectExists(JobListing $entity, string $langcode): /** * Create a redirect for the indexing request. * + * Temporary redirect is created for all entities before requesting indexing. + * Once delete request is sent, the url cannot be indexed again using the api. + * Hence we should not use the original url. + * * @param JobListing $entity * The entity to index. * @param string $langcode - * + * The language code. * * @return string - * Absolute url that can be send for indexing. + * Absolute url that can be sent for indexing. */ private function createTemporaryRedirectUrl(JobListing $entity, string $langcode): string { $alias = $this->getEntityAlias($entity, $langcode); @@ -167,6 +252,17 @@ private function createTemporaryRedirectUrl(JobListing $entity, string $langcode return $indexing_url; } + /** + * Get the temporary redirect url. + * + * @param JobListing $entity + * The entity to index. + * @param string $langcode + * The language code. + * + * @return Redirect|null + * The redirect object. + */ private function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { $job_alias = $this->getEntityAlias($entity, $langcode); @@ -190,18 +286,21 @@ private function getExistingTemporaryRedirect(JobListing $entity, string $langco return NULL; } + /** + * Get the alias for an entity. + * + * @param JobListing $entity + * The entity. + * @param string $langcode + * The language code. + * + * @return string + * Alias for the entity. + */ private function getEntityAlias(JobListing $entity, string $langcode): string { return $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $langcode); } - public function checkItemIndexStatus(string $url): string { - try { - return $this->helfiGoogleApi->checkIndexingStatus($url); - } - catch (GuzzleException $e) { - throw new \Exception($e->getMessage(), $e->getCode()); - } - } /** * Update any jobs that were published today. From b833c51f981a3717d0dcebac2138bd484ef187f4 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Thu, 5 Sep 2024 14:04:53 +0300 Subject: [PATCH 12/57] UHF-8706: added temp redirect delete, removed useless code --- .../src/Drush/Commands/HelfiApiCommands.php | 4 +- .../helfi_google_api/src/HelfiGoogleApi.php | 1 - .../src/JobIndexingService.php | 172 ++---------------- 3 files changed, 14 insertions(+), 163 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 9e4b004e..732fff09 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -11,11 +11,9 @@ use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\node\Entity\Node; use Drupal\path_alias\AliasManagerInterface; -use Drupal\redirect\Entity\Redirect; use Drush\Attributes\Command; use Drush\Commands\AutowireTrait; use Drush\Commands\DrushCommands; -use Symfony\Component\HttpFoundation\RedirectResponse; /** * A Drush command file. @@ -45,7 +43,7 @@ public function __construct( * @return int * The exit code. */ - #[Command(name: 'helfi:single-index-google')] + #[Command(name: 'helfi:google-single-entity-index')] public function indexSingleItem( int $entity_id, string $langcode = 'fi', diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index 7137e27c..fdd759f8 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -93,7 +93,6 @@ public function indexBatch(array $urls, bool $update): array { $responses = $batch->execute(); $errors = []; - foreach ($responses as $key => $response) { if ($response instanceof Exception) { $errors[] = "$key: {$response->getMessage()}"; diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index dc6f0904..9cdf7e0b 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -114,7 +114,14 @@ public function deindexEntity(JobListing $entity): array { $url_to_deindex = $base_url . $redirect->getSourceUrl(); - return $this->deindexItems([$url_to_deindex]); + try { + $response = $this->deindexItems([$url_to_deindex]); + $redirect->delete(); + return $response; + } catch(GuzzleException $e) { + $this->logger->error("Error while deindexing entity id {$entity->id()}: {$e->getMessage()}"); + throw new \Exception('Failed de-index redirect.'); + } } /** @@ -263,7 +270,7 @@ private function createTemporaryRedirectUrl(JobListing $entity, string $langcode * @return Redirect|null * The redirect object. */ - private function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { + public function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { $job_alias = $this->getEntityAlias($entity, $langcode); $query = $this->entityTypeManager->getStorage('redirect') @@ -276,6 +283,10 @@ private function getExistingTemporaryRedirect(JobListing $entity, string $langco ->execute(); $redirects = Redirect::loadMultiple($redirectIds); + if (!$redirects) { + return NULL; + } + foreach ($redirects as $redirect) { $source = $redirect->getSourceUrl(); @@ -301,161 +312,4 @@ private function getEntityAlias(JobListing $entity, string $langcode): string { return $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $langcode); } - - /** - * Update any jobs that were published today. - * - * @return void - */ - public function indexJobs(): void { - $storage = $this->entityTypeManager - ->getStorage('node'); - $startOfToday = strtotime('today 00:00:00'); - $endOfToday = strtotime('today 23:59:59'); - - // Etsi nodet joiden publish_on on tänään - // TAI ei oo tänään mutta on luotu tänään - - $jobIds = $storage->getQuery() - // ->condition('publish_on', $startOfToday, '>=') - // ->condition('publish_on', $endOfToday, '<=') - ->condition('status', 1) - ->condition('type', 'job_listing') - ->latestRevision() - ->accessCheck(FALSE) - ->execute(); - - if (!$jobIds) { - return; - } - - $jobs = $storage->loadMultiple($jobIds); - $urls = $this->prepareUrls($jobs); - - $this->helfiGoogleApi->indexJobsBatch($urls); - } - - public function deleteIndexedJobs() { - - $storage = $this->entityTypeManager - ->getStorage('node'); - $startOfToday = strtotime('today 00:00:00'); - $endOfToday = strtotime('today 23:59:59'); - - // Etsi nodet joiden unpublish on on tänään - $jobIds = $storage->getQuery() - ->condition('unpublish_on', $startOfToday, '>=') - ->condition('unpublish_on', $endOfToday, '<=') - ->condition('status', 0) - ->condition('type', 'job_listing') - ->latestRevision() - ->accessCheck(FALSE) - ->execute(); - - if (!$jobIds) { - return; - } - - $jobs = $storage->loadMultiple($jobIds); - - // hae redirectit - $redirect = $this->getRedirects($jobs); - - // kaiva se urli sieltä - // lähetä googlelle - // poista redirect - - // $this->helfiGoogleApi->indexJobsBatch($urls); - // $urls = $this->getUrls($jobs); - - // Redirect::load() - - } - - private function prepareUrls(array $jobs, array $langcodes = ['fi', 'en', 'sv']): array { - $result = []; - - $now = strtotime('now'); - - foreach ($jobs as $job) { - - foreach($langcodes as $langcode) { - if ($job->hasTranslation($langcode)) { - $job = $job->getTranslation($langcode); - $alias = $this->aliasManager->getAliasByPath("/node/{$job->id()}", $langcode); - $temp_alias = "{$alias}-{$now}"; - - $redirect = Redirect::create([ - 'redirect_source' => ltrim($temp_alias, '/'), - 'redirect_redirect' => "internal:/node/{$job->id()}", - 'language' => $langcode, - 'status_code' => 301, - 'title' => $job->getTitle(), - ]); - - // $redirect->setRedirect( "/node/{$job->id()}", [], ['redirect_title' => $job->getTitle()]); - $save_status = $redirect->save(); - - if ($save_status === SAVED_NEW) { - $index_url = "{$job->toUrl()->setAbsolute(TRUE)->toString()}-{$now}"; - $result[] = $index_url; - } - - } - } - } - - return $result; - } - - /** - * Delete a temporary redirect. - * - * @param array $jobs - * Array of nodes. - * - * @return array - */ - private function deleteRedirect(array $jobs): array { - $result = []; - $langcodes = ['fi', 'en', 'sv']; - - foreach ($jobs as $job) { - foreach($langcodes as $langcode) { - if ($job->hasTranslation($langcode)) { - $job = $job->getTranslation($langcode); - - $redirectIds = \Drupal::entityQuery('redirect') - ->condition('redirect_redirect__uri', "internal:/node/{$job->id()}") - ->condition('status_code', 301) - ->condition('language', $langcode) - ->execute(); - - $redirects = Redirect::loadMultiple($redirectIds); - /** @var Redirect $redirect */ - foreach ($redirects as $redirect) { - $alias = $this->aliasManager->getAliasByPath("/node/{$job->id()}", $langcode); - $source = $redirect->getSourceUrl(); - if (!str_contains($source, "$alias-")) { - continue; - } - - $result[] = $source; - try { - $redirect->delete(); - } - catch(\Exception $e) { - // Add logging. - } - - } - } - } - - return $result; - } - - return $result; - } - } From 3736ef35bc0453cf49aa3b112c84f73b26085372 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 08:47:14 +0300 Subject: [PATCH 13/57] UHF-8706: added command for remove-request, code fixes --- .../helfi_google_api.services.yml | 6 +- .../src/Drush/Commands/HelfiApiCommands.php | 35 +++++++ .../JobPublishStateSubscriber.php | 4 +- .../helfi_google_api/src/HelfiGoogleApi.php | 13 +++ .../src/JobIndexingService.php | 98 ++++++++++++------- 5 files changed, 116 insertions(+), 40 deletions(-) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index 80207837..be5ff453 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -15,4 +15,8 @@ services: arguments: $logger: '@logger.channel.helfi_google_api' - Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: ~ + Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: + class: Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber + autowire: true + arguments: + $logger: '@logger.channel.helfi_google_api' diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 732fff09..a20cfe12 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -78,6 +78,41 @@ public function indexSingleItem( return DrushCommands::EXIT_SUCCESS; } + #[Command(name: 'helfi:google-single-entity-deindex')] + public function deindexSingleItem( + int $entity_id, + string $langcode = 'fi', + ) : int { + $entity = Node::load($entity_id); + + if (!$entity instanceof JobListing) { + $this->io()->writeln('Entity not found or not instance of JobListing'); + return DrushCommands::EXIT_FAILURE; + } + + if (!$entity->hasTranslation($langcode)) { + $this->io()->writeln('Translation does not exist.'); + return DrushCommands::EXIT_FAILURE; + } + $entity = $entity->getTranslation($langcode); + + try { + $response = $this->jobIndexingService->deindexEntity($entity); + } + catch (\Exception $e) { + $this->io()->error($e->getMessage()); + return DrushCommands::EXIT_FAILURE; + } + + if ($response['errors']) { + $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response['errors'])); + return DrushCommands::EXIT_FAILURE_WITH_CLARITY; + } + + $this->io()->writeln('Url deindexed succesfully.'); + return DrushCommands::EXIT_SUCCESS; + } + /** * Request url indexing status from Google api. * diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index cde622c8..5dbdff36 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -6,10 +6,8 @@ use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; -use Drupal\redirect\Entity\Redirect; use Drupal\scheduler\SchedulerEvent; use Drupal\scheduler\SchedulerEvents; -use Drush\Commands\DrushCommands; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** @@ -19,6 +17,7 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { public function __construct( private readonly JobIndexingService $jobIndexingService, + private readonly LoggerInterface $logger, ) { } @@ -91,7 +90,6 @@ public function sendDeindexingRequest(SchedulerEvent $event) { $redirect = $this->jobIndexingService->getExistingTemporaryRedirect($entity, $langcode); if (!$redirect) { - // Log, the item seems not to be indexed. return; } diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index fdd759f8..401d84a3 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -58,6 +58,18 @@ public function __construct( ) { } + /** + * Does the api key exist. + * + * @return bool + * Api key exists. + */ + public function hasAuthenticationKey(): bool { + $config = $this->configFactory->get('helfi_google_api.settings'); + $key = $config->get('indexing_api_key') ?: ''; + return !!$key; + } + /** * Send indexing or deindexing request for urls. * @@ -124,6 +136,7 @@ public function checkIndexingStatus($url): string { $the_url = $base_url . $queryParameter; $client = $this->indexingService->getClient()->authorize(); + $response = $client->get($the_url); return $response->getBody()->getContents(); diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 9cdf7e0b..9116897d 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -51,11 +51,12 @@ public function indexItems(array $urls): array { * Array of containing total count indexed and errors. */ public function indexEntity(JobListing $entity): array { + if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + throw new \Exception('Api key not set.'); + } + $langcode = $entity->language()->getId(); - $results = []; - // If the actual url is deindexed, it can't be reindexed again. - // If entity has a temporary redirect, it most likely has already been indexed already. $hasRedirect = $this->temporaryRedirectExists($entity, $langcode); if ($hasRedirect) { throw new \Exception('Already indexed.'); @@ -64,17 +65,25 @@ public function indexEntity(JobListing $entity): array { // Create temporary redirect for the entity. $indexing_url = $this->createTemporaryRedirectUrl($entity, $langcode); - $result = $this->indexItems([$indexing_url]); + try { + $result = $this->indexItems([$indexing_url]); + } + catch (GuzzleException $e) { + $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; + $this->logger->error($message); + throw new \Exception($message); + } if ($result['errors']) { - // Some of the urls failed - // log + $total = $result['count']; + $error_count = count($result['errors']); + $error_string = json_encode($result['errors']); + $this->logger->error("Unable to index $error_count/$total items: $error_string"); } return $result; } - /** * Send urls to Google for deindexing. * @@ -84,7 +93,7 @@ public function indexEntity(JobListing $entity): array { * @return array * Array: 'count': int, 'errors': array. */ - public function deindexItems(array $urls): array { + public function deindexItems(array $urls): array { return $this->helfiGoogleApi->indexBatch($urls, FALSE); } @@ -98,30 +107,45 @@ public function deindexItems(array $urls): array { * Array: 'count': int, 'errors': array */ public function deindexEntity(JobListing $entity): array { + if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + throw new \Exception('Api key not set.'); + } + $language = $entity->language(); $redirect = $this->getExistingTemporaryRedirect($entity, $language->getId()); if (!$redirect) { - // Log, the item seems not to be indexed. - // Return. - throw new \Exception(); + $this->logger->error("Entity of id {$entity->id()} doesn't have the required temporary redirect."); + throw new \Exception("Entity of id {$entity->id()} doesn't have the required temporary redirect."); } $base_url = $this->urlGenerator->generateFromRoute( '', [], - ['absolute' => TRUE,'language' => $language] + [ + 'absolute' => TRUE, + 'language' => $language, + ] ); $url_to_deindex = $base_url . $redirect->getSourceUrl(); try { - $response = $this->deindexItems([$url_to_deindex]); - $redirect->delete(); - return $response; - } catch(GuzzleException $e) { - $this->logger->error("Error while deindexing entity id {$entity->id()}: {$e->getMessage()}"); - throw new \Exception('Failed de-index redirect.'); + $result = $this->deindexItems([$url_to_deindex]); } + catch(GuzzleException $e) { + $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}" + $this->logger->error($message); + throw new \Exception($message); + } + + if ($result['errors']) { + $total = $result['count']; + $error_count = count($result['errors']); + $error_string = json_encode($result['errors']); + $this->logger->error("Unable to index $error_count/$total items: $error_string"); + } + + return $result; } /** @@ -134,21 +158,22 @@ public function deindexEntity(JobListing $entity): array { * Status as a string. */ public function checkItemIndexStatus(string $url): string { - try { - return $this->helfiGoogleApi->checkIndexingStatus($url); - } - catch (GuzzleException $e) { - throw new \Exception($e->getMessage(), $e->getCode()); - } + return $this->helfiGoogleApi->checkIndexingStatus($url); } /** * If entity seems to be indexed, send a status query. * * @param JobListing $entity + * Entity to check. + * * @return array */ public function checkEntityIndexStatus(JobListing $entity): string { + if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + throw new \Exception('Api key not set.'); + } + $language = $entity->language(); $baseUrl = $this->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); @@ -173,30 +198,31 @@ public function checkEntityIndexStatus(JobListing $entity): string { } if (!$correct_redirect) { - // HAS NOT BEEN SENT AFAIK + throw new \Exception('Entity doesn\'t have temporary redirect.'); } $url_to_check = $baseUrl . $correct_redirect->getSourceUrl(); try { $response = $this->helfiGoogleApi->checkIndexingStatus($url_to_check); } + catch(GuzzleException $e) { + $this->logger->error("Request failed with code {$e->getCode()}: {$e->getMessage()}"); + } catch (\Exception $e) { - // REQUEST FAILED - + $this->logger->error('Error while checking indexing status: ' . $e->getMessage()); } return $response; - } /** * Does the entity have a temporary redirect. * * Temporary redirect is created for all entities before requesting indexing. - * Once delete request is sent, the url cannot be indexed again using the api. + * Once delete-request is sent, the url cannot be indexed again using the api. * Hence we should not use the original url. * - * @param JobListing $entity + * @param Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to check. * @param string $langcode * The language code. @@ -204,7 +230,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { * @return bool * Has temporary redirect. */ - private function temporaryRedirectExists(JobListing $entity, string $langcode): bool { + public function temporaryRedirectExists(JobListing $entity, string $langcode): bool { $job_alias = $this->getEntityAlias($entity, $langcode); $query = $this->entityTypeManager->getStorage('redirect') @@ -235,7 +261,7 @@ private function temporaryRedirectExists(JobListing $entity, string $langcode): * Once delete request is sent, the url cannot be indexed again using the api. * Hence we should not use the original url. * - * @param JobListing $entity + * @param Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to index. * @param string $langcode * The language code. @@ -243,7 +269,7 @@ private function temporaryRedirectExists(JobListing $entity, string $langcode): * @return string * Absolute url that can be sent for indexing. */ - private function createTemporaryRedirectUrl(JobListing $entity, string $langcode): string { + public function createTemporaryRedirectUrl(JobListing $entity, string $langcode): string { $alias = $this->getEntityAlias($entity, $langcode); $now = strtotime('now'); $temp_alias = "$alias-$now"; @@ -262,12 +288,12 @@ private function createTemporaryRedirectUrl(JobListing $entity, string $langcode /** * Get the temporary redirect url. * - * @param JobListing $entity + * @param Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to index. * @param string $langcode * The language code. * - * @return Redirect|null + * @return Drupal\redirect\Entity\Redirect|null * The redirect object. */ public function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { @@ -300,7 +326,7 @@ public function getExistingTemporaryRedirect(JobListing $entity, string $langcod /** * Get the alias for an entity. * - * @param JobListing $entity + * @param Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity. * @param string $langcode * The language code. From 0f75547d82eaa133af9285f35127066d098441b6 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 09:12:21 +0300 Subject: [PATCH 14/57] UHF-8706: override the key from env variables --- public/sites/default/production.settings.php | 1 + 1 file changed, 1 insertion(+) diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index 7a1ea057..da9b0d4c 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -2,3 +2,4 @@ $config['openid_connect.client.tunnistamo']['settings']['is_production'] = TRUE; $config['helfi_proxy.settings']['tunnistamo_return_url'] = '/fi/avoimet-tyopaikat/openid-connect/tunnistamo'; +$config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE-INDEXING-API-KEY'); From dce918398b2026cb0f666a68c14a1758d5aa97c0 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 09:19:10 +0300 Subject: [PATCH 15/57] UHF-8706: get rid of \Drupal call --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 5dbdff36..cbc81a32 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -4,6 +4,7 @@ namespace Drupal\helfi_google_api\EventSubscriber; +use Drupal\Core\Routing\UrlGeneratorInterface; use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; @@ -18,6 +19,7 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { public function __construct( private readonly JobIndexingService $jobIndexingService, private readonly LoggerInterface $logger, + private readonly UrlGeneratorInterface $urlGenerator, ) { } @@ -79,7 +81,7 @@ public function sendIndexingRequest(SchedulerEvent $event) { * Send deindexing request to google. * * @param SchedulerEvent $event - * @return void + * The scheduler event. */ public function sendDeindexingRequest(SchedulerEvent $event) { $entity = $event->getNode(); @@ -93,7 +95,7 @@ public function sendDeindexingRequest(SchedulerEvent $event) { return; } - $base_url = \Drupal::urlGenerator()->generateFromRoute( + $base_url = $this->urlGenerator->generateFromRoute( '', [], ['absolute' => TRUE,'language' => $entity->language()] From 213dad0fb090c8d42fea04e55ce9582352902835 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 09:34:40 +0300 Subject: [PATCH 16/57] UHF-8706: code fixes, changed eventsubscriber to use the better service method --- .../src/Drush/Commands/HelfiApiCommands.php | 11 ++++ .../JobPublishStateSubscriber.php | 64 ++++--------------- .../src/JobIndexingService.php | 12 ++-- 3 files changed, 28 insertions(+), 59 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index a20cfe12..de3d9adb 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -78,6 +78,17 @@ public function indexSingleItem( return DrushCommands::EXIT_SUCCESS; } + /** + * Deindex single entity by id, + * + * @param int $entity_id + * The entity id. + * @param string $langcode + * The entity langcode. + * + * @return int + * The exit code. + */ #[Command(name: 'helfi:google-single-entity-deindex')] public function deindexSingleItem( int $entity_id, diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index cbc81a32..cc939a29 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -23,7 +23,6 @@ public function __construct( ) { } - /** * {@inheritdoc} */ @@ -35,75 +34,34 @@ public static function getSubscribedEvents() { ]; } - public function sendIndexingRequest(SchedulerEvent $event) { + /** + * Send indexing request to google. + * + * @param \Drupal\scheduler\SchedulerEvent $event + * The scheduler event. + */ + public function sendIndexingRequest(SchedulerEvent $event): void { $entity = $event->getNode(); if (!$entity instanceof JobListing) { return; } - $langcodes = ['fi', 'en', 'sv']; - $results = []; - - foreach($langcodes as $langcode) { - try { - $hasRedirect = $this->jobIndexingService->temporaryRedirectExists($entity, $langcode); - if ($hasRedirect) { - // log, continue. - continue; - } - } - catch (\Exception $e) { - // Log. - continue; - } - - try { - $indexing_url = $this->jobIndexingService->createTemporaryRedirectUrl($entity, $langcode); - } - catch (\Exception $e) { - // cannot create url, log - continue; - } - - $results[] = $indexing_url; - } - - $result = $this->jobIndexingService->indexItems($results); - - if ($result['errors']) { - // Some of the urls failed - // log - } - + $this->jobIndexingService->indexEntity($entity); } /** * Send deindexing request to google. * - * @param SchedulerEvent $event + * @param \Drupal\scheduler\SchedulerEvent $event * The scheduler event. */ - public function sendDeindexingRequest(SchedulerEvent $event) { + public function sendDeindexingRequest(SchedulerEvent $event): void { $entity = $event->getNode(); - $langcode = $entity->language()->getId(); if (!$entity instanceof JobListing) { return; } - $redirect = $this->jobIndexingService->getExistingTemporaryRedirect($entity, $langcode); - if (!$redirect) { - return; - } - - $base_url = $this->urlGenerator->generateFromRoute( - '', - [], - ['absolute' => TRUE,'language' => $entity->language()] - ); - - $url_to_deindex = $base_url . $redirect->getSourceUrl(); - - $this->jobIndexingService->deindexItems([$url_to_deindex]); + $this->jobIndexingService->deindexEntity($entity); } } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 9116897d..116e7551 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -98,9 +98,9 @@ public function deindexItems(array $urls): array { } /** - * Handle entity indexing request + * Handle entity indexing request. * - * @param JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to request deindexing for. * * @return array @@ -132,8 +132,8 @@ public function deindexEntity(JobListing $entity): array { try { $result = $this->deindexItems([$url_to_deindex]); } - catch(GuzzleException $e) { - $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}" + catch (GuzzleException $e) { + $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; $this->logger->error($message); throw new \Exception($message); } @@ -164,7 +164,7 @@ public function checkItemIndexStatus(string $url): string { /** * If entity seems to be indexed, send a status query. * - * @param JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to check. * * @return array @@ -205,7 +205,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { try { $response = $this->helfiGoogleApi->checkIndexingStatus($url_to_check); } - catch(GuzzleException $e) { + catch (GuzzleException $e) { $this->logger->error("Request failed with code {$e->getCode()}: {$e->getMessage()}"); } catch (\Exception $e) { From 40527c4c120c8c9df66863b4a584cdfea84a3cd0 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 09:48:28 +0300 Subject: [PATCH 17/57] UHF-8706: code fixes --- public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php | 4 +--- .../custom/helfi_google_api/src/JobIndexingService.php | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index 401d84a3..9b990c89 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -128,7 +128,7 @@ public function indexBatch(array $urls, bool $update): array { * @return string * The response as a string. */ - public function checkIndexingStatus($url): string { + public function checkIndexingStatus(string $url): string { $this->initializeApi(FALSE); $base_url = self::METADATA_ENDPOINT; @@ -147,8 +147,6 @@ public function checkIndexingStatus($url): string { * * @param bool $batch * Set the client on batch mode. - * - * @return void */ private function initializeApi(bool $batch = TRUE): void { $config = $this->configFactory->get('helfi_google_api.settings'); diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 116e7551..30cd060e 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -167,7 +167,8 @@ public function checkItemIndexStatus(string $url): string { * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to check. * - * @return array + * @return string + * The url index status as a string. */ public function checkEntityIndexStatus(JobListing $entity): string { if (!$this->helfiGoogleApi->hasAuthenticationKey()) { From f4f4a67f968456b19fcb367debc882e4a809f363 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 09:56:16 +0300 Subject: [PATCH 18/57] UHF-8706: comment must end with a full stop --- .../helfi_google_api/src/Drush/Commands/HelfiApiCommands.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index de3d9adb..a1e8c7e2 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -79,7 +79,7 @@ public function indexSingleItem( } /** - * Deindex single entity by id, + * Deindex single entity by id. * * @param int $entity_id * The entity id. From 3d527ee9edfa3f8b01fba2c034f0a215276f6b53 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:03:28 +0300 Subject: [PATCH 19/57] UHF-8706: urlGenerator was missing for some reason --- .../modules/custom/helfi_google_api/src/JobIndexingService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 30cd060e..03c6f8a8 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -177,7 +177,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { $language = $entity->language(); - $baseUrl = $this->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); + $baseUrl = $this->urlGenerator->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); $job_alias = $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $language->getId()); $query = $this->entityTypeManager->getStorage('redirect')->getQuery(); From 092d488e2b1f06e2467a6db758804d055e23cb2e Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:37:57 +0300 Subject: [PATCH 20/57] UHF-8706: phpstan fixes --- .../src/Drush/Commands/HelfiApiCommands.php | 2 +- .../JobPublishStateSubscriber.php | 9 ++++++--- .../helfi_google_api/src/HelfiGoogleApi.php | 6 ++---- .../src/JobIndexingService.php | 19 ++++++++++--------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index a1e8c7e2..8c183a68 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -127,7 +127,7 @@ public function deindexSingleItem( /** * Request url indexing status from Google api. * - * @param int $url + * @param string $url * The url to check. * * @return int diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index cc939a29..522b2cf6 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -4,7 +4,6 @@ namespace Drupal\helfi_google_api\EventSubscriber; -use Drupal\Core\Routing\UrlGeneratorInterface; use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; @@ -16,10 +15,14 @@ */ class JobPublishStateSubscriber implements EventSubscriberInterface { + /** + * The constructor. + * + * @param \Drupal\helfi_google_api\JobIndexingService $jobIndexingService + * The job indexing service. + */ public function __construct( private readonly JobIndexingService $jobIndexingService, - private readonly LoggerInterface $logger, - private readonly UrlGeneratorInterface $urlGenerator, ) { } diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index 9b990c89..6f2df12c 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -42,15 +42,13 @@ class HelfiGoogleApi { /** * Google indexing service. - * - * @var Google\Service\Indexing */ private Indexing $indexingService; /** * The constructor. * - * @param Drupal\Core\Config\ConfigFactoryInterface $configFactory + * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory * The config factory. */ public function __construct( @@ -137,7 +135,7 @@ public function checkIndexingStatus(string $url): string { $client = $this->indexingService->getClient()->authorize(); - $response = $client->get($the_url); + $response = $client->request('GET', $the_url); return $response->getBody()->getContents(); } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 03c6f8a8..705db219 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -44,7 +44,7 @@ public function indexItems(array $urls): array { /** * Send indexing request to google. * - * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity which indexing should be requested. * * @return array @@ -198,22 +198,23 @@ public function checkEntityIndexStatus(JobListing $entity): string { } } - if (!$correct_redirect) { + if (!isset($correct_redirect)) { throw new \Exception('Entity doesn\'t have temporary redirect.'); } $url_to_check = $baseUrl . $correct_redirect->getSourceUrl(); try { - $response = $this->helfiGoogleApi->checkIndexingStatus($url_to_check); + return $this->helfiGoogleApi->checkIndexingStatus($url_to_check); } catch (GuzzleException $e) { $this->logger->error("Request failed with code {$e->getCode()}: {$e->getMessage()}"); + throw new \Exception("Request failed with code {$e->getCode()}: {$e->getMessage()}"); } catch (\Exception $e) { $this->logger->error('Error while checking indexing status: ' . $e->getMessage()); + throw new \Exception("Something went wrong: {$e->getMessage()}"); } - return $response; } /** @@ -223,7 +224,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { * Once delete-request is sent, the url cannot be indexed again using the api. * Hence we should not use the original url. * - * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to check. * @param string $langcode * The language code. @@ -262,7 +263,7 @@ public function temporaryRedirectExists(JobListing $entity, string $langcode): b * Once delete request is sent, the url cannot be indexed again using the api. * Hence we should not use the original url. * - * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to index. * @param string $langcode * The language code. @@ -289,12 +290,12 @@ public function createTemporaryRedirectUrl(JobListing $entity, string $langcode) /** * Get the temporary redirect url. * - * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity to index. * @param string $langcode * The language code. * - * @return Drupal\redirect\Entity\Redirect|null + * @return \Drupal\redirect\Entity\Redirect|null * The redirect object. */ public function getExistingTemporaryRedirect(JobListing $entity, string $langcode): Redirect|null { @@ -327,7 +328,7 @@ public function getExistingTemporaryRedirect(JobListing $entity, string $langcod /** * Get the alias for an entity. * - * @param Drupal\helfi_rekry_content\Entity\JobListing $entity + * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * The entity. * @param string $langcode * The language code. From 69bb44b79db30d6698e9c399e15107f74e35492b Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:42:27 +0300 Subject: [PATCH 21/57] UHF-8706: no need for logger here any more --- .../custom/helfi_google_api/helfi_google_api.services.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index be5ff453..5ea053f6 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -9,11 +9,7 @@ services: Drupal\helfi_google_api\HelfiGoogleApi: ~ - Drupal\helfi_google_api\JobIndexingService: - class: Drupal\helfi_google_api\JobIndexingService - autowire: true - arguments: - $logger: '@logger.channel.helfi_google_api' + Drupal\helfi_google_api\JobIndexingService: ~ Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: class: Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber From 01a8ad5647d5e7ba0ff8839c247ed4595d497a55 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:45:39 +0300 Subject: [PATCH 22/57] UHF-8706: fix services yml --- .../custom/helfi_google_api/helfi_google_api.services.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index 5ea053f6..80207837 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -9,10 +9,10 @@ services: Drupal\helfi_google_api\HelfiGoogleApi: ~ - Drupal\helfi_google_api\JobIndexingService: ~ - - Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: - class: Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber + Drupal\helfi_google_api\JobIndexingService: + class: Drupal\helfi_google_api\JobIndexingService autowire: true arguments: $logger: '@logger.channel.helfi_google_api' + + Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: ~ From c55c478f411d0df41cdcacc96e5f06c1b24ee6e8 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:57:01 +0300 Subject: [PATCH 23/57] UHF-8706: less duplication --- .../src/JobIndexingService.php | 49 ++++++------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 705db219..bbbc4056 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -29,16 +29,25 @@ public function __construct( } /** - * Send urls to google for indexing. + * Send urls to Google for deindexing. * * @param array $urls - * Array of urls to index. + * Urls to remove from index. + * @param bool $update + * Send update or delete request (indexing or deindexing). * * @return array - * Array of containing total count indexed and errors. + * Array: 'count': int, 'errors': array. */ - public function indexItems(array $urls): array { - return $this->helfiGoogleApi->indexBatch($urls, TRUE); + public function handleIndexingRequest(array $urls, bool $update): array { + try { + return $this->helfiGoogleApi->indexBatch($urls, $update); + } + catch (GuzzleException $e) { + $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; + $this->logger->error($message); + throw new \Exception($message); + } } /** @@ -65,14 +74,7 @@ public function indexEntity(JobListing $entity): array { // Create temporary redirect for the entity. $indexing_url = $this->createTemporaryRedirectUrl($entity, $langcode); - try { - $result = $this->indexItems([$indexing_url]); - } - catch (GuzzleException $e) { - $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; - $this->logger->error($message); - throw new \Exception($message); - } + $result = $this->handleIndexingRequest([$indexing_url], TRUE); if ($result['errors']) { $total = $result['count']; @@ -84,18 +86,6 @@ public function indexEntity(JobListing $entity): array { return $result; } - /** - * Send urls to Google for deindexing. - * - * @param array $urls - * Urls to remove from index. - * - * @return array - * Array: 'count': int, 'errors': array. - */ - public function deindexItems(array $urls): array { - return $this->helfiGoogleApi->indexBatch($urls, FALSE); - } /** * Handle entity indexing request. @@ -129,14 +119,7 @@ public function deindexEntity(JobListing $entity): array { $url_to_deindex = $base_url . $redirect->getSourceUrl(); - try { - $result = $this->deindexItems([$url_to_deindex]); - } - catch (GuzzleException $e) { - $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; - $this->logger->error($message); - throw new \Exception($message); - } + $result = $this->handleIndexingRequest([$url_to_deindex], FALSE); if ($result['errors']) { $total = $result['count']; From 2ca0d968621d1c1154861fd3166410caa53c158b Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 10:59:01 +0300 Subject: [PATCH 24/57] UHF-8706: remove extra lines --- .../modules/custom/helfi_google_api/src/JobIndexingService.php | 1 - 1 file changed, 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index bbbc4056..874f587a 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -86,7 +86,6 @@ public function indexEntity(JobListing $entity): array { return $result; } - /** * Handle entity indexing request. * From 2f699876da84049dedf2769d6a5bb825e3b35674 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 11:02:32 +0300 Subject: [PATCH 25/57] UHF-8706: use bool cast instead of double exlcamation --- public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php index 6f2df12c..81e05282 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php @@ -65,7 +65,7 @@ public function __construct( public function hasAuthenticationKey(): bool { $config = $this->configFactory->get('helfi_google_api.settings'); $key = $config->get('indexing_api_key') ?: ''; - return !!$key; + return (bool) $key; } /** From 2fd0513e08799ec76d9defb6be2f8db5c63b7dc8 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 13:55:21 +0300 Subject: [PATCH 26/57] UHF-8706: qa fixes --- .../src/Drush/Commands/HelfiApiCommands.php | 4 ++-- .../JobPublishStateSubscriber.php | 2 +- .../src/{HelfiGoogleApi.php => GoogleApi.php} | 10 +++++----- .../src/JobIndexingService.php | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) rename public/modules/custom/helfi_google_api/src/{HelfiGoogleApi.php => GoogleApi.php} (94%) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 8c183a68..b93a5c1f 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -134,7 +134,7 @@ public function deindexSingleItem( * The exit code. */ #[Command(name: 'helfi:google-url-index-status')] - public function checkUrlIndexStatus(string $url) { + public function checkUrlIndexStatus(string $url): int { try { $response = $this->jobIndexingService->checkItemIndexStatus($url); } @@ -159,7 +159,7 @@ public function checkUrlIndexStatus(string $url) { * The exit code. */ #[Command(name: 'helfi:google-entity-index-status')] - public function checkEntityIndexStatus(int $entity_id, $langcode = 'fi') { + public function checkEntityIndexStatus(int $entity_id, $langcode = 'fi'): int { $entity = Node::load($entity_id); if (!$entity instanceof JobListing) { diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 522b2cf6..05e09e17 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -29,7 +29,7 @@ public function __construct( /** * {@inheritdoc} */ - public static function getSubscribedEvents() { + public static function getSubscribedEvents(): array { return [ SchedulerEvents::PUBLISH => 'sendIndexingRequest', SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', diff --git a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php similarity index 94% rename from public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php rename to public/modules/custom/helfi_google_api/src/GoogleApi.php index 81e05282..3083067c 100644 --- a/public/modules/custom/helfi_google_api/src/HelfiGoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -13,7 +13,7 @@ /** * A wrapper for Google indexing library. */ -class HelfiGoogleApi { +class GoogleApi { /** * Endpoint for sending update and delete requests. @@ -129,13 +129,13 @@ public function indexBatch(array $urls, bool $update): array { public function checkIndexingStatus(string $url): string { $this->initializeApi(FALSE); - $base_url = self::METADATA_ENDPOINT; - $queryParameter = '?url=' . urlencode($url); - $the_url = $base_url . $queryParameter; + $baseUrl = self::METADATA_ENDPOINT; + $query_parameter = '?url=' . urlencode($url); + $theUrl = $baseUrl . $query_parameter; $client = $this->indexingService->getClient()->authorize(); - $response = $client->request('GET', $the_url); + $response = $client->request('GET', $theUrl); return $response->getBody()->getContents(); } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 874f587a..a7e677db 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -20,7 +20,7 @@ class JobIndexingService { use AutowireTrait; public function __construct( - private readonly HelfiGoogleApi $helfiGoogleApi, + private readonly GoogleApi $googleApi, private readonly EntityTypeManagerInterface $entityTypeManager, private readonly AliasManagerInterface $aliasManager, private readonly UrlGeneratorInterface $urlGenerator, @@ -41,7 +41,7 @@ public function __construct( */ public function handleIndexingRequest(array $urls, bool $update): array { try { - return $this->helfiGoogleApi->indexBatch($urls, $update); + return $this->googleApi->indexBatch($urls, $update); } catch (GuzzleException $e) { $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; @@ -60,7 +60,7 @@ public function handleIndexingRequest(array $urls, bool $update): array { * Array of containing total count indexed and errors. */ public function indexEntity(JobListing $entity): array { - if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + if (!$this->googleApi->hasAuthenticationKey()) { throw new \Exception('Api key not set.'); } @@ -96,7 +96,7 @@ public function indexEntity(JobListing $entity): array { * Array: 'count': int, 'errors': array */ public function deindexEntity(JobListing $entity): array { - if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + if (!$this->googleApi->hasAuthenticationKey()) { throw new \Exception('Api key not set.'); } @@ -140,7 +140,7 @@ public function deindexEntity(JobListing $entity): array { * Status as a string. */ public function checkItemIndexStatus(string $url): string { - return $this->helfiGoogleApi->checkIndexingStatus($url); + return $this->googleApi->checkIndexingStatus($url); } /** @@ -153,7 +153,7 @@ public function checkItemIndexStatus(string $url): string { * The url index status as a string. */ public function checkEntityIndexStatus(JobListing $entity): string { - if (!$this->helfiGoogleApi->hasAuthenticationKey()) { + if (!$this->googleApi->hasAuthenticationKey()) { throw new \Exception('Api key not set.'); } @@ -186,15 +186,15 @@ public function checkEntityIndexStatus(JobListing $entity): string { $url_to_check = $baseUrl . $correct_redirect->getSourceUrl(); try { - return $this->helfiGoogleApi->checkIndexingStatus($url_to_check); + return $this->googleApi->checkIndexingStatus($url_to_check); } catch (GuzzleException $e) { $this->logger->error("Request failed with code {$e->getCode()}: {$e->getMessage()}"); - throw new \Exception("Request failed with code {$e->getCode()}: {$e->getMessage()}"); + throw $e; } catch (\Exception $e) { $this->logger->error('Error while checking indexing status: ' . $e->getMessage()); - throw new \Exception("Something went wrong: {$e->getMessage()}"); + throw $e; } } From cd9c38c94f439e9756ebb84f6ab9e91be8ea984c Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 13:57:04 +0300 Subject: [PATCH 27/57] UHF-8706: key underscored --- public/sites/default/production.settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index da9b0d4c..a1cd0f16 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -2,4 +2,4 @@ $config['openid_connect.client.tunnistamo']['settings']['is_production'] = TRUE; $config['helfi_proxy.settings']['tunnistamo_return_url'] = '/fi/avoimet-tyopaikat/openid-connect/tunnistamo'; -$config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE-INDEXING-API-KEY'); +$config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE_INDEXING_API_KEY'); From f963a65a15c128cd3b47edf35c2e281d81d40f17 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Fri, 6 Sep 2024 14:00:00 +0300 Subject: [PATCH 28/57] UHF-8706: correct service name --- .../custom/helfi_google_api/helfi_google_api.services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index 80207837..ef8b79aa 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -7,7 +7,7 @@ services: parent: logger.channel_base arguments: ['helfi_google_api'] - Drupal\helfi_google_api\HelfiGoogleApi: ~ + Drupal\helfi_google_api\GoogleApi: ~ Drupal\helfi_google_api\JobIndexingService: class: Drupal\helfi_google_api\JobIndexingService From 5bc5d99b7463dc6df3fc1bc3d883abe9b6b0d7dd Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 08:32:36 +0300 Subject: [PATCH 29/57] UHF-8706: delete redirect on deindex, delete redirect on failed index request, added documentation --- .../modules/custom/helfi_google_api/README.md | 47 +++++++++++++++++++ .../helfi_google_api.info.yml | 1 + .../JobPublishStateSubscriber.php | 15 +++++- .../src/JobIndexingService.php | 39 ++++++++++----- 4 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 public/modules/custom/helfi_google_api/README.md diff --git a/public/modules/custom/helfi_google_api/README.md b/public/modules/custom/helfi_google_api/README.md new file mode 100644 index 00000000..97308d32 --- /dev/null +++ b/public/modules/custom/helfi_google_api/README.md @@ -0,0 +1,47 @@ +# Google indexing api + +The module handles job listing indexing and deindexing using API provided by Google. Module requires an api key to work. +Api key is only set to production environment since Google doesn't provide testing environment. + +* `drupal/scheduler` -module events are used to trigger the indexing requests. +* `google/apiclient` -library is used to handle the communication with Google api. + +Api documentation: https://developers.google.com/search/apis/indexing-api/v3/quickstart +Google library: https://github.com/googleapis/google-api-php-client +Api key: Check keyvault + +## Development / local testing + +### API KEY +You must set the api key in local.settings.php in order to use the module. Without it, the feature won't do anything at all. +Api key is set to local.settings.php like this: `$config['helfi_google_api.settings']['indexing_api_key'] = '{}'` +Instead of empty json object you can get the correct key from Keyvault. + +### Indexing requests + +The indexing api only allows sending urls pointing to hel.fi domain. Therefore you can properly test the features +on production environment. You can send production urls to indexing api from local environment if you have +the auth key set properly. + +Sending local url to google indexing api results in 4xx error. + + +## Requests + +### Indexing + +* Indexing request is tied to scheduler event +* A temporary redirect is created for the entity that is used for the indexing. + +### Deindexing + +* Dendexing request is tied to scheduler event +* The temporary redirect is removed when deindexing is done. + +### Status check + +* You can send a status request to google api to find out if an URL has been indexed or deleted throuhg the API + + + + diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml index ecd47da5..32fe007c 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml @@ -6,3 +6,4 @@ core_version_requirement: ^10 || ^11 dependencies: - 'publication_date:publication_date' - redirect:redirect + - scheduler:scheduler diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 05e09e17..0f25106b 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -30,6 +30,7 @@ public function __construct( * {@inheritdoc} */ public static function getSubscribedEvents(): array { + // @todo Enable when tested with cron commands. return [ SchedulerEvents::PUBLISH => 'sendIndexingRequest', SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', @@ -49,7 +50,12 @@ public function sendIndexingRequest(SchedulerEvent $event): void { return; } - $this->jobIndexingService->indexEntity($entity); + try { + $this->jobIndexingService->indexEntity($entity); + } + catch(\Exception $exception){ + // has been logged by indexing service. + } } /** @@ -64,7 +70,12 @@ public function sendDeindexingRequest(SchedulerEvent $event): void { return; } - $this->jobIndexingService->deindexEntity($entity); + try { + $this->jobIndexingService->deindexEntity($entity); + } + catch(\Exception) { + // Has been logged by indexing service. + } } } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index a7e677db..7e1a7593 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -29,10 +29,10 @@ public function __construct( } /** - * Send urls to Google for deindexing. + * Send urls to Google for indexing or deindexing. * * @param array $urls - * Urls to remove from index. + * Urls to add or remove from index. * @param bool $update * Send update or delete request (indexing or deindexing). * @@ -72,9 +72,18 @@ public function indexEntity(JobListing $entity): array { } // Create temporary redirect for the entity. - $indexing_url = $this->createTemporaryRedirectUrl($entity, $langcode); + $redirectArray = $this->createTemporaryRedirectUrl($entity, $langcode); + $indexing_url = $redirectArray['indexing_url']; - $result = $this->handleIndexingRequest([$indexing_url], TRUE); + try { + $result = $this->handleIndexingRequest([$indexing_url], TRUE); + } + catch (\Exception $e) { + // If the request fails, remove the redirect. + $redirect = $redirectArray['redirect']; + $redirect->delete(); + throw $e; + } if ($result['errors']) { $total = $result['count']; @@ -87,7 +96,7 @@ public function indexEntity(JobListing $entity): array { } /** - * Handle entity indexing request. + * Handle entity deindexing request. * * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to request deindexing for. @@ -117,8 +126,14 @@ public function deindexEntity(JobListing $entity): array { ); $url_to_deindex = $base_url . $redirect->getSourceUrl(); + try { + $result = $this->handleIndexingRequest([$url_to_deindex], FALSE); + } + catch (\Exception $e) { + throw $e; + } - $result = $this->handleIndexingRequest([$url_to_deindex], FALSE); + $redirect->delete(); if ($result['errors']) { $total = $result['count']; @@ -190,7 +205,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { } catch (GuzzleException $e) { $this->logger->error("Request failed with code {$e->getCode()}: {$e->getMessage()}"); - throw $e; + throw new \Exception($e->getMessage()); } catch (\Exception $e) { $this->logger->error('Error while checking indexing status: ' . $e->getMessage()); @@ -250,23 +265,23 @@ public function temporaryRedirectExists(JobListing $entity, string $langcode): b * @param string $langcode * The language code. * - * @return string - * Absolute url that can be sent for indexing. + * @return array + * Indexing_url as the absolute url and the redirect object. */ - public function createTemporaryRedirectUrl(JobListing $entity, string $langcode): string { + public function createTemporaryRedirectUrl(JobListing $entity, string $langcode): array { $alias = $this->getEntityAlias($entity, $langcode); $now = strtotime('now'); $temp_alias = "$alias-$now"; $indexing_url = "{$entity->toUrl()->setAbsolute()->toString()}-$now"; - Redirect::create([ + $redirect = Redirect::create([ 'redirect_source' => ltrim($temp_alias, '/'), 'redirect_redirect' => "internal:/node/{$entity->id()}", 'language' => $langcode, 'status_code' => 301, ])->save(); - return $indexing_url; + return ['indexing_url' => $indexing_url, 'redirect' => $redirect]; } /** From c563be5e841faa025a388792318144513b90e77f Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 08:42:28 +0300 Subject: [PATCH 30/57] UHF-8706: twig security update --- composer.lock | 175 +++++++++++++++++++++++++------------------------- 1 file changed, 87 insertions(+), 88 deletions(-) diff --git a/composer.lock b/composer.lock index f11d3d8f..b3c42f15 100644 --- a/composer.lock +++ b/composer.lock @@ -13366,86 +13366,6 @@ ], "time": "2024-05-31T15:07:36+00:00" }, - { - "name": "symfony/polyfill-php80", - "version": "v1.30.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "77fa7995ac1b21ab60769b7323d600a991a90433" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/77fa7995ac1b21ab60769b7323d600a991a90433", - "reference": "77fa7995ac1b21ab60769b7323d600a991a90433", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "type": "library", - "extra": { - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Ion Bazan", - "email": "ion.bazan@gmail.com" - }, - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.30.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-05-31T15:07:36+00:00" - }, { "name": "symfony/polyfill-php81", "version": "v1.30.0", @@ -14601,24 +14521,23 @@ }, { "name": "twig/twig", - "version": "v3.11.0", + "version": "v3.14.0", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "e80fb8ebba85c7341a97a9ebf825d7fd4b77708d" + "reference": "126b2c97818dbff0cdf3fbfc881aedb3d40aae72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/e80fb8ebba85c7341a97a9ebf825d7fd4b77708d", - "reference": "e80fb8ebba85c7341a97a9ebf825d7fd4b77708d", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/126b2c97818dbff0cdf3fbfc881aedb3d40aae72", + "reference": "126b2c97818dbff0cdf3fbfc881aedb3d40aae72", "shasum": "" }, "require": { - "php": ">=7.2.5", + "php": ">=8.0.2", "symfony/deprecation-contracts": "^2.5|^3", "symfony/polyfill-ctype": "^1.8", "symfony/polyfill-mbstring": "^1.3", - "symfony/polyfill-php80": "^1.22", "symfony/polyfill-php81": "^1.29" }, "require-dev": { @@ -14665,7 +14584,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.11.0" + "source": "https://github.com/twigphp/Twig/tree/v3.14.0" }, "funding": [ { @@ -14677,7 +14596,7 @@ "type": "tidelift" } ], - "time": "2024-08-08T16:15:16+00:00" + "time": "2024-09-09T17:55:12+00:00" }, { "name": "twistor/flysystem-stream-wrapper", @@ -19903,6 +19822,86 @@ ], "time": "2024-01-23T14:51:35+00:00" }, + { + "name": "symfony/polyfill-php80", + "version": "v1.30.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php80.git", + "reference": "77fa7995ac1b21ab60769b7323d600a991a90433" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/77fa7995ac1b21ab60769b7323d600a991a90433", + "reference": "77fa7995ac1b21ab60769b7323d600a991a90433", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php80\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ion Bazan", + "email": "ion.bazan@gmail.com" + }, + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php80/tree/v1.30.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-05-31T15:07:36+00:00" + }, { "name": "symfony/polyfill-php82", "version": "v1.29.0", From 1fc3c1d192b0510291a40241af8a9f36df49d6e9 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 08:46:52 +0300 Subject: [PATCH 31/57] UHF-8706: code fixes --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 0f25106b..f7c39066 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -53,8 +53,8 @@ public function sendIndexingRequest(SchedulerEvent $event): void { try { $this->jobIndexingService->indexEntity($entity); } - catch(\Exception $exception){ - // has been logged by indexing service. + catch (\Exception $exception) { + // Has been logged by indexing service. } } @@ -73,7 +73,7 @@ public function sendDeindexingRequest(SchedulerEvent $event): void { try { $this->jobIndexingService->deindexEntity($entity); } - catch(\Exception) { + catch (\Exception) { // Has been logged by indexing service. } } From 3b8520b93793355b2238c2b71df4d1eb511ef8fb Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 09:16:30 +0300 Subject: [PATCH 32/57] UHF-8706: typos --- public/modules/custom/helfi_google_api/README.md | 4 ++-- .../custom/helfi_google_api/tests/src/Kernel/IndexingTest.php | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 public/modules/custom/helfi_google_api/tests/src/Kernel/IndexingTest.php diff --git a/public/modules/custom/helfi_google_api/README.md b/public/modules/custom/helfi_google_api/README.md index 97308d32..7a7f11ae 100644 --- a/public/modules/custom/helfi_google_api/README.md +++ b/public/modules/custom/helfi_google_api/README.md @@ -35,12 +35,12 @@ Sending local url to google indexing api results in 4xx error. ### Deindexing -* Dendexing request is tied to scheduler event +* Deindexing request is tied to scheduler event * The temporary redirect is removed when deindexing is done. ### Status check -* You can send a status request to google api to find out if an URL has been indexed or deleted throuhg the API +* You can send a status request to google api to find out if an URL has been indexed or deleted through the API diff --git a/public/modules/custom/helfi_google_api/tests/src/Kernel/IndexingTest.php b/public/modules/custom/helfi_google_api/tests/src/Kernel/IndexingTest.php new file mode 100644 index 00000000..e69de29b From a0f446e9dffbf3bce23d02c75412c605894db787 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 10:16:51 +0300 Subject: [PATCH 33/57] UHF-8706: we don't want this to run in any environment until this has been tested --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index f7c39066..d1de6027 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -30,12 +30,15 @@ public function __construct( * {@inheritdoc} */ public static function getSubscribedEvents(): array { - // @todo Enable when tested with cron commands. + // @todo Enable after feature tested in production. + /* return [ - SchedulerEvents::PUBLISH => 'sendIndexingRequest', - SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', - SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', + SchedulerEvents::PUBLISH => 'sendIndexingRequest', + SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', + SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', ]; + */ + return []; } /** From f713f076b183d2d91b37d355ef47fe2c46e0175b Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 10:24:47 +0300 Subject: [PATCH 34/57] UHF-8706: unused use statement --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 1 - 1 file changed, 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index d1de6027..9945be73 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -7,7 +7,6 @@ use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; -use Drupal\scheduler\SchedulerEvents; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** From 4b0eba42f4982528a9c7df81bc18fb705d71559e Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 10:51:28 +0300 Subject: [PATCH 35/57] UHF-8706: run only in production. added in a mention in main readme --- .../helfi_google_api.info.yml | 1 + .../JobPublishStateSubscriber.php | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml index 32fe007c..45315f1b 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml @@ -7,3 +7,4 @@ dependencies: - 'publication_date:publication_date' - redirect:redirect - scheduler:scheduler + - helfi_api_base:helfi_api_base diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 9945be73..461f5029 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -4,6 +4,8 @@ namespace Drupal\helfi_google_api\EventSubscriber; +use Drupal\helfi_api_base\Environment\EnvironmentEnum; +use Drupal\helfi_api_base\Environment\EnvironmentResolverInterface; use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; @@ -22,6 +24,7 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { */ public function __construct( private readonly JobIndexingService $jobIndexingService, + private readonly EnvironmentResolverInterface $environmentResolver, ) { } @@ -47,6 +50,10 @@ public static function getSubscribedEvents(): array { * The scheduler event. */ public function sendIndexingRequest(SchedulerEvent $event): void { + if (!$this->isProduction()) { + return; + } + $entity = $event->getNode(); if (!$entity instanceof JobListing) { return; @@ -67,6 +74,10 @@ public function sendIndexingRequest(SchedulerEvent $event): void { * The scheduler event. */ public function sendDeindexingRequest(SchedulerEvent $event): void { + if (!$this->isProduction()) { + return; + } + $entity = $event->getNode(); if (!$entity instanceof JobListing) { return; @@ -80,4 +91,14 @@ public function sendDeindexingRequest(SchedulerEvent $event): void { } } + /** + * Check if in production. + * + * @return bool + * Is production. + */ + private function isProduction(): bool { + return $this->environmentResolver->getActiveEnvironment()->getEnvironment() === EnvironmentEnum::Prod; + } + } From 00264a9fc8da1754fc73ed1a02caaedacc0ea1c5 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 11:25:19 +0300 Subject: [PATCH 36/57] UHF-8706: add the readme update --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f8d57895..fbfd1195 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,11 @@ local site. 4. To test the functionality, you should use the Mailpit running on your local in the `https://mailpit.docker.so/` url to view the emails being sent by the feature. +#### Google indexing api automation (helfi_google_api-module) + +Job listing urls are automatically sent to google indexing api +on publish and unpublish events, a request is sent to google to either index or deindex the url. + ## Customizations ### Not part of global navigation From 41c7927aac5a0a9373d90acca21c24b648d960bc Mon Sep 17 00:00:00 2001 From: rpnykanen <48206082+rpnykanen@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:46:39 +0300 Subject: [PATCH 37/57] added env resolver to docblock --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 461f5029..13cebba5 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -21,6 +21,8 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { * * @param \Drupal\helfi_google_api\JobIndexingService $jobIndexingService * The job indexing service. + * @param Drupal\helfi_api_base\Environment\EnvironmentResolverInterface $environmentResolver + * The environment resolver. */ public function __construct( private readonly JobIndexingService $jobIndexingService, From 9d6a56df6f9678ae11bfa349f6b56f59f3cf9351 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 14:50:48 +0300 Subject: [PATCH 38/57] UHF-8706: added missing comma --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 13cebba5..94a2add0 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -21,7 +21,7 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { * * @param \Drupal\helfi_google_api\JobIndexingService $jobIndexingService * The job indexing service. - * @param Drupal\helfi_api_base\Environment\EnvironmentResolverInterface $environmentResolver + * @param \Drupal\helfi_api_base\Environment\EnvironmentResolverInterface $environmentResolver * The environment resolver. */ public function __construct( From d4f255807d9580e2c66a22d2aac50dfedc2a749b Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 18:31:48 +0300 Subject: [PATCH 39/57] UHF-8706: added factory and updated services --- .../helfi_google_api.services.yml | 18 ++++++- .../src/GoogleServiceFactory.php | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml index ef8b79aa..9f389276 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.services.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.services.yml @@ -7,7 +7,11 @@ services: parent: logger.channel_base arguments: ['helfi_google_api'] - Drupal\helfi_google_api\GoogleApi: ~ + Drupal\helfi_google_api\GoogleApi: + class: Drupal\helfi_google_api\GoogleApi + arguments: + - '@config.factory' + - '@helfi_google_api.google_service' Drupal\helfi_google_api\JobIndexingService: class: Drupal\helfi_google_api\JobIndexingService @@ -16,3 +20,15 @@ services: $logger: '@logger.channel.helfi_google_api' Drupal\helfi_google_api\EventSubscriber\JobPublishStateSubscriber: ~ + + helfi_google_api.google_service: + public: false + class: \Google\Service\Indexing + factory: [ '@helfi_google_api.google_service_factory', 'create'] + arguments: + - '@config.factory' + + helfi_google_api.google_service_factory: + class: Drupal\helfi_google_api\GoogleServiceFactory + arguments: + - '@config.factory' diff --git a/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php b/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php new file mode 100644 index 00000000..790fab3f --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php @@ -0,0 +1,48 @@ +get('helfi_google_api.settings'); + $key = $config->get('indexing_api_key') ?: ''; + + $client = new GoogleClient(); + $client->setApplicationName('Helfi_Rekry'); + $client->addScope(self::SCOPES); + $client->setUseBatch(TRUE); + + // Set empty json object as a key if necessary. + if ($key) { + $client->setAuthConfig(json_decode($key, TRUE)); + $client->authorize(); + } + + return new Indexing($client); + } + +} From 6fb22eef1b039ef93d5c467b6d192c34952901e2 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 18:32:07 +0300 Subject: [PATCH 40/57] UHF-8706: added simple response class --- .../custom/helfi_google_api/src/Response.php | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 public/modules/custom/helfi_google_api/src/Response.php diff --git a/public/modules/custom/helfi_google_api/src/Response.php b/public/modules/custom/helfi_google_api/src/Response.php new file mode 100644 index 00000000..4b98b7a6 --- /dev/null +++ b/public/modules/custom/helfi_google_api/src/Response.php @@ -0,0 +1,62 @@ +urls; + } + + /** + * Get the errors. + * + * @return array + * Errors for each url. + */ + public function getErrors(): array { + return $this->errors; + } + + /** + * The request was not actually sent. + * + * Either the api key is not set or + * in settings config, enabled in false. + * + * @return bool + * This is a debug run. + */ + public function isDebug(): bool { + return $this->debug; + } + +} From c5ebd3d32f6903a2662be48950dd6a234eb062dc Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 18:34:50 +0300 Subject: [PATCH 41/57] UHF-8706: get rid of the simple prod check --- .../JobPublishStateSubscriber.php | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 94a2add0..9945be73 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -4,8 +4,6 @@ namespace Drupal\helfi_google_api\EventSubscriber; -use Drupal\helfi_api_base\Environment\EnvironmentEnum; -use Drupal\helfi_api_base\Environment\EnvironmentResolverInterface; use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; @@ -21,12 +19,9 @@ class JobPublishStateSubscriber implements EventSubscriberInterface { * * @param \Drupal\helfi_google_api\JobIndexingService $jobIndexingService * The job indexing service. - * @param \Drupal\helfi_api_base\Environment\EnvironmentResolverInterface $environmentResolver - * The environment resolver. */ public function __construct( private readonly JobIndexingService $jobIndexingService, - private readonly EnvironmentResolverInterface $environmentResolver, ) { } @@ -52,10 +47,6 @@ public static function getSubscribedEvents(): array { * The scheduler event. */ public function sendIndexingRequest(SchedulerEvent $event): void { - if (!$this->isProduction()) { - return; - } - $entity = $event->getNode(); if (!$entity instanceof JobListing) { return; @@ -76,10 +67,6 @@ public function sendIndexingRequest(SchedulerEvent $event): void { * The scheduler event. */ public function sendDeindexingRequest(SchedulerEvent $event): void { - if (!$this->isProduction()) { - return; - } - $entity = $event->getNode(); if (!$entity instanceof JobListing) { return; @@ -93,14 +80,4 @@ public function sendDeindexingRequest(SchedulerEvent $event): void { } } - /** - * Check if in production. - * - * @return bool - * Is production. - */ - private function isProduction(): bool { - return $this->environmentResolver->getActiveEnvironment()->getEnvironment() === EnvironmentEnum::Prod; - } - } From 24e8c240fb6be969539503af4139d2a5d487c702 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 19:00:35 +0300 Subject: [PATCH 42/57] UHF-8706: prevent sending actual request if key is not set or enabled is falsey for debugging, added debug logic --- conf/cmi/helfi_google_api.settings.yml | 1 + .../helfi_google_api.info.yml | 1 - .../src/Drush/Commands/HelfiApiCommands.php | 20 ++++- .../custom/helfi_google_api/src/GoogleApi.php | 82 ++++++++----------- .../src/JobIndexingService.php | 78 +++++++++--------- public/sites/default/production.settings.php | 1 + 6 files changed, 91 insertions(+), 92 deletions(-) diff --git a/conf/cmi/helfi_google_api.settings.yml b/conf/cmi/helfi_google_api.settings.yml index 8407ee2e..db0e4c95 100644 --- a/conf/cmi/helfi_google_api.settings.yml +++ b/conf/cmi/helfi_google_api.settings.yml @@ -1 +1,2 @@ +enabled: false indexing_api_key: '' diff --git a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml index 45315f1b..32fe007c 100644 --- a/public/modules/custom/helfi_google_api/helfi_google_api.info.yml +++ b/public/modules/custom/helfi_google_api/helfi_google_api.info.yml @@ -7,4 +7,3 @@ dependencies: - 'publication_date:publication_date' - redirect:redirect - scheduler:scheduler - - helfi_api_base:helfi_api_base diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index b93a5c1f..ec412b7a 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -69,11 +69,17 @@ public function indexSingleItem( return DrushCommands::EXIT_FAILURE; } - if ($response['errors']) { - $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response['errors'])); + if ($response->getErrors()) { + $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response->getErrors())); return DrushCommands::EXIT_FAILURE_WITH_CLARITY; } + if ($response->isDebug()) { + $urls = $response->getUrls(); + $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); + return DrushCommands::EXIT_SUCCESS; + } + $this->io()->writeln('Url indexed succesfully.'); return DrushCommands::EXIT_SUCCESS; } @@ -115,11 +121,17 @@ public function deindexSingleItem( return DrushCommands::EXIT_FAILURE; } - if ($response['errors']) { - $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response['errors'])); + if ($response->getErrors()) { + $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response->getErrors())); return DrushCommands::EXIT_FAILURE_WITH_CLARITY; } + if ($response->isDebug()) { + $urls = $response->getUrls(); + $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); + return DrushCommands::EXIT_SUCCESS; + } + $this->io()->writeln('Url deindexed succesfully.'); return DrushCommands::EXIT_SUCCESS; } diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 3083067c..492e882d 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -5,7 +5,6 @@ namespace Drupal\helfi_google_api; use Drupal\Core\Config\ConfigFactoryInterface; -use Google\Client as GoogleClient; use Google\Service\Exception; use Google\Service\Indexing; use GuzzleHttp\Psr7\Request; @@ -25,11 +24,6 @@ class GoogleApi { */ const METADATA_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications/metadata'; - /** - * Api scopes. - */ - const SCOPES = ['https://www.googleapis.com/auth/indexing']; - /** * Request type for update-request (indexing). */ @@ -40,19 +34,17 @@ class GoogleApi { */ const DELETE = 'URL_DELETED'; - /** - * Google indexing service. - */ - private Indexing $indexingService; - /** * The constructor. * * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory * The config factory. + * @param \Google\Service\Indexing $indexingService + * The Google indexing service. */ public function __construct( private readonly ConfigFactoryInterface $configFactory, + private readonly Indexing $indexingService, ) { } @@ -68,6 +60,20 @@ public function hasAuthenticationKey(): bool { return (bool) $key; } + /** + * Correct environment and key is set. + * + * @return bool + * The api is set up + */ + public function isEnabled(): bool { + $config = $this->configFactory->get('helfi_google_api.settings'); + $key = $config->get('indexing_api_key') ?: ''; + $isEnabled = $config->get('enabled') ?: FALSE; + + return $key && $isEnabled; + } + /** * Send indexing or deindexing request for urls. * @@ -76,11 +82,14 @@ public function hasAuthenticationKey(): bool { * @param bool $update * TRUE to index the urls, FALSE for deindexing. * - * @return array - * Array which consists of the amount of items sent and array of errors. + * @return Response + * Object which holds the handled urls and request errors. */ - public function indexBatch(array $urls, bool $update): array { - $this->initializeApi(); + public function indexBatch(array $urls, bool $update): Response { + if (!$this->isEnabled()) { + return new Response($urls, debug: TRUE); + } + $batch = $this->indexingService->createBatch(); $operation = $update ? self::UPDATE : self::DELETE; @@ -109,10 +118,7 @@ public function indexBatch(array $urls, bool $update): array { } } - return [ - 'count' => count($urls), - 'errors' => $errors, - ]; + return new Response($urls, $errors); } /** @@ -127,41 +133,19 @@ public function indexBatch(array $urls, bool $update): array { * The response as a string. */ public function checkIndexingStatus(string $url): string { - $this->initializeApi(FALSE); + $client = $this->indexingService->getClient(); + $client->setUseBatch(FALSE); + + if ($this->isEnabled()) { + $client = $client->authorize(); + } $baseUrl = self::METADATA_ENDPOINT; $query_parameter = '?url=' . urlencode($url); $theUrl = $baseUrl . $query_parameter; - $client = $this->indexingService->getClient()->authorize(); - - $response = $client->request('GET', $theUrl); - - return $response->getBody()->getContents(); - } - - /** - * Set up the client. - * - * @param bool $batch - * Set the client on batch mode. - */ - private function initializeApi(bool $batch = TRUE): void { - $config = $this->configFactory->get('helfi_google_api.settings'); - $key = $config->get('indexing_api_key') ?: ''; - - if (!$key) { - throw new \Exception('Api key not configured. Unable to proceed.'); - } - - $client = new GoogleClient(); - $client->setApplicationName('Helfi_Rekry'); - $client->setAuthConfig(json_decode($key, TRUE)); - - $client->addScope(self::SCOPES); - $client->setUseBatch($batch); - - $this->indexingService = new Indexing($client); + $result = $client->request('GET', $theUrl); + return $result->getBody()->getContents(); } } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 7e1a7593..50cde600 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -36,10 +36,10 @@ public function __construct( * @param bool $update * Send update or delete request (indexing or deindexing). * - * @return array - * Array: 'count': int, 'errors': array. + * @return \Drupal\helfi_google_api\Response + * The response object. */ - public function handleIndexingRequest(array $urls, bool $update): array { + public function handleIndexingRequest(array $urls, bool $update): Response { try { return $this->googleApi->indexBatch($urls, $update); } @@ -56,17 +56,13 @@ public function handleIndexingRequest(array $urls, bool $update): array { * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity which indexing should be requested. * - * @return array - * Array of containing total count indexed and errors. + * @return \Drupal\helfi_google_api\Response + * The response object. */ - public function indexEntity(JobListing $entity): array { - if (!$this->googleApi->hasAuthenticationKey()) { - throw new \Exception('Api key not set.'); - } - + public function indexEntity(JobListing $entity): Response { $langcode = $entity->language()->getId(); - $hasRedirect = $this->temporaryRedirectExists($entity, $langcode); + $hasRedirect = $this->hasTemporaryRedirect($entity, $langcode); if ($hasRedirect) { throw new \Exception('Already indexed.'); } @@ -74,22 +70,22 @@ public function indexEntity(JobListing $entity): array { // Create temporary redirect for the entity. $redirectArray = $this->createTemporaryRedirectUrl($entity, $langcode); $indexing_url = $redirectArray['indexing_url']; + $redirect = $redirectArray['redirect']; try { $result = $this->handleIndexingRequest([$indexing_url], TRUE); } catch (\Exception $e) { // If the request fails, remove the redirect. - $redirect = $redirectArray['redirect']; $redirect->delete(); throw $e; } - if ($result['errors']) { - $total = $result['count']; - $error_count = count($result['errors']); - $error_string = json_encode($result['errors']); - $this->logger->error("Unable to index $error_count/$total items: $error_string"); + if ($result->getErrors()) { + $total = count($result->getUrls()); + $errorCount = count($result->getErrors()); + $errorsString = json_encode($result->getErrors()); + $this->logger->error(("Unable to index $errorCount/$total items: $errorsString")); } return $result; @@ -101,19 +97,16 @@ public function indexEntity(JobListing $entity): array { * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to request deindexing for. * - * @return array - * Array: 'count': int, 'errors': array + * @return \Drupal\helfi_google_api\Response + * The response object. */ - public function deindexEntity(JobListing $entity): array { - if (!$this->googleApi->hasAuthenticationKey()) { - throw new \Exception('Api key not set.'); - } - + public function deindexEntity(JobListing $entity): Response { $language = $entity->language(); $redirect = $this->getExistingTemporaryRedirect($entity, $language->getId()); if (!$redirect) { - $this->logger->error("Entity of id {$entity->id()} doesn't have the required temporary redirect."); - throw new \Exception("Entity of id {$entity->id()} doesn't have the required temporary redirect."); + $message = "Entity of id {$entity->id()} doesn't have the required temporary redirect."; + $this->logger->error($message); + throw new \Exception($message); } $base_url = $this->urlGenerator->generateFromRoute( @@ -133,13 +126,16 @@ public function deindexEntity(JobListing $entity): array { throw $e; } - $redirect->delete(); + // No need to delete redirects on debug run. + if (!$result->isDebug()) { + $redirect->delete(); + } - if ($result['errors']) { - $total = $result['count']; - $error_count = count($result['errors']); - $error_string = json_encode($result['errors']); - $this->logger->error("Unable to index $error_count/$total items: $error_string"); + if ($result->getErrors()) { + $total = count($result->getUrls()); + $errorCount = count($result->getErrors()); + $errorsString = json_encode($result->getErrors()); + $this->logger->error(("Unable to index $errorCount/$total items: $errorsString")); } return $result; @@ -168,10 +164,6 @@ public function checkItemIndexStatus(string $url): string { * The url index status as a string. */ public function checkEntityIndexStatus(JobListing $entity): string { - if (!$this->googleApi->hasAuthenticationKey()) { - throw new \Exception('Api key not set.'); - } - $language = $entity->language(); $baseUrl = $this->urlGenerator->generateFromRoute('', [], ['absolute' => TRUE, 'language' => $language]); @@ -229,7 +221,7 @@ public function checkEntityIndexStatus(JobListing $entity): string { * @return bool * Has temporary redirect. */ - public function temporaryRedirectExists(JobListing $entity, string $langcode): bool { + public function hasTemporaryRedirect(JobListing $entity, string $langcode): bool { $job_alias = $this->getEntityAlias($entity, $langcode); $query = $this->entityTypeManager->getStorage('redirect') @@ -279,7 +271,12 @@ public function createTemporaryRedirectUrl(JobListing $entity, string $langcode) 'redirect_redirect' => "internal:/node/{$entity->id()}", 'language' => $langcode, 'status_code' => 301, - ])->save(); + ]); + + // If the api is not set up, no need to create the redirect. + if ($this->googleApi->isEnabled()) { + $redirect->save(); + } return ['indexing_url' => $indexing_url, 'redirect' => $redirect]; } @@ -308,6 +305,11 @@ public function getExistingTemporaryRedirect(JobListing $entity, string $langcod ->execute(); $redirects = Redirect::loadMultiple($redirectIds); + // For debugging purposes, dbugging won't save the redirect. + if (!$redirects && !$this->googleApi->isEnabled()) { + return $this->createTemporaryRedirectUrl($entity, $langcode)['redirect']; + } + if (!$redirects) { return NULL; } diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index a1cd0f16..6cde48f0 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -3,3 +3,4 @@ $config['openid_connect.client.tunnistamo']['settings']['is_production'] = TRUE; $config['helfi_proxy.settings']['tunnistamo_return_url'] = '/fi/avoimet-tyopaikat/openid-connect/tunnistamo'; $config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE_INDEXING_API_KEY'); +$config['helfi_google_api.settings']['enabled'] = getenv('APP_ENV') === 'production'; From c571b242e0a28df3e5fae85bc454af8ac5026333 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 19:19:24 +0300 Subject: [PATCH 43/57] UHF-8706: less duplication --- .../src/Drush/Commands/HelfiApiCommands.php | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index ec412b7a..6dda11bd 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -14,6 +14,7 @@ use Drush\Attributes\Command; use Drush\Commands\AutowireTrait; use Drush\Commands\DrushCommands; +use Drupal\helfi_google_api\Response; /** * A Drush command file. @@ -69,19 +70,7 @@ public function indexSingleItem( return DrushCommands::EXIT_FAILURE; } - if ($response->getErrors()) { - $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response->getErrors())); - return DrushCommands::EXIT_FAILURE_WITH_CLARITY; - } - - if ($response->isDebug()) { - $urls = $response->getUrls(); - $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); - return DrushCommands::EXIT_SUCCESS; - } - - $this->io()->writeln('Url indexed succesfully.'); - return DrushCommands::EXIT_SUCCESS; + return $this->handleResponse($response); } /** @@ -121,19 +110,7 @@ public function deindexSingleItem( return DrushCommands::EXIT_FAILURE; } - if ($response->getErrors()) { - $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response->getErrors())); - return DrushCommands::EXIT_FAILURE_WITH_CLARITY; - } - - if ($response->isDebug()) { - $urls = $response->getUrls(); - $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); - return DrushCommands::EXIT_SUCCESS; - } - - $this->io()->writeln('Url deindexed succesfully.'); - return DrushCommands::EXIT_SUCCESS; + return $this->handleResponse($response); } /** @@ -197,4 +174,29 @@ public function checkEntityIndexStatus(int $entity_id, $langcode = 'fi'): int { return DrushCommands::EXIT_SUCCESS; } + /** + * Handle response. + * + * @param Drupal\helfi_google_api\Response $response + * The response object. + * + * @return int + * Exit code. + */ + private function handleResponse(Response $response): int { + if ($response->getErrors()) { + $this->io()->writeln('Request successful. Errors returned: ' . json_encode($response->getErrors())); + return DrushCommands::EXIT_FAILURE_WITH_CLARITY; + } + + if ($response->isDebug()) { + $urls = $response->getUrls(); + $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); + return DrushCommands::EXIT_SUCCESS; + } + + $this->io()->writeln('Url indexed succesfully.'); + return DrushCommands::EXIT_SUCCESS; + } + } From c59ce0bf84204da728f8e6dcb8ca4be208229e1a Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Tue, 10 Sep 2024 19:22:43 +0300 Subject: [PATCH 44/57] UHF-8706: code fixes --- .../helfi_google_api/src/Drush/Commands/HelfiApiCommands.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php index 6dda11bd..ee478731 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php @@ -8,13 +8,13 @@ use Drupal\Core\Language\LanguageManagerInterface; use Drupal\Core\Routing\UrlGeneratorInterface; use Drupal\helfi_google_api\JobIndexingService; +use Drupal\helfi_google_api\Response; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\node\Entity\Node; use Drupal\path_alias\AliasManagerInterface; use Drush\Attributes\Command; use Drush\Commands\AutowireTrait; use Drush\Commands\DrushCommands; -use Drupal\helfi_google_api\Response; /** * A Drush command file. @@ -177,7 +177,7 @@ public function checkEntityIndexStatus(int $entity_id, $langcode = 'fi'): int { /** * Handle response. * - * @param Drupal\helfi_google_api\Response $response + * @param \Drupal\helfi_google_api\Response $response * The response object. * * @return int From 5ce41b8e0bb3afbad7a885eeaaafb492ce9288ae Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 09:20:51 +0300 Subject: [PATCH 45/57] UHF-8706: renamed function with better name --- public/modules/custom/helfi_google_api/src/GoogleApi.php | 6 +++--- .../custom/helfi_google_api/src/JobIndexingService.php | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 492e882d..1a6658da 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -66,7 +66,7 @@ public function hasAuthenticationKey(): bool { * @return bool * The api is set up */ - public function isEnabled(): bool { + public function isDryRun(): bool { $config = $this->configFactory->get('helfi_google_api.settings'); $key = $config->get('indexing_api_key') ?: ''; $isEnabled = $config->get('enabled') ?: FALSE; @@ -86,7 +86,7 @@ public function isEnabled(): bool { * Object which holds the handled urls and request errors. */ public function indexBatch(array $urls, bool $update): Response { - if (!$this->isEnabled()) { + if (!$this->isDryRun()) { return new Response($urls, debug: TRUE); } @@ -136,7 +136,7 @@ public function checkIndexingStatus(string $url): string { $client = $this->indexingService->getClient(); $client->setUseBatch(FALSE); - if ($this->isEnabled()) { + if ($this->isDryRun()) { $client = $client->authorize(); } diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 50cde600..33ec0b3b 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -274,7 +274,7 @@ public function createTemporaryRedirectUrl(JobListing $entity, string $langcode) ]); // If the api is not set up, no need to create the redirect. - if ($this->googleApi->isEnabled()) { + if ($this->googleApi->isDryRun()) { $redirect->save(); } @@ -306,7 +306,7 @@ public function getExistingTemporaryRedirect(JobListing $entity, string $langcod $redirects = Redirect::loadMultiple($redirectIds); // For debugging purposes, dbugging won't save the redirect. - if (!$redirects && !$this->googleApi->isEnabled()) { + if (!$redirects && !$this->googleApi->isDryRun()) { return $this->createTemporaryRedirectUrl($entity, $langcode)['redirect']; } From 5b15b08458789b676a464064d6555ea496e67c65 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 09:22:41 +0300 Subject: [PATCH 46/57] UHF-8706: remove comment from eventsubscriber --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 9945be73..ebb59e92 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -7,6 +7,7 @@ use Drupal\helfi_google_api\JobIndexingService; use Drupal\helfi_rekry_content\Entity\JobListing; use Drupal\scheduler\SchedulerEvent; +use Drupal\scheduler\SchedulerEvents; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** @@ -29,14 +30,12 @@ public function __construct( * {@inheritdoc} */ public static function getSubscribedEvents(): array { - // @todo Enable after feature tested in production. - /* return [ - SchedulerEvents::PUBLISH => 'sendIndexingRequest', - SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', - SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', + SchedulerEvents::PUBLISH => 'sendIndexingRequest', + SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', + SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', ]; - */ + return []; } From c2a157582ee2f13e1e863a30a0665e5deab7909e Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 09:25:30 +0300 Subject: [PATCH 47/57] UHF-8706: class description --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index ebb59e92..5994e6bf 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -11,7 +11,7 @@ use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** - * {@inheritdoc} + * Subscribe to job publishing events */ class JobPublishStateSubscriber implements EventSubscriberInterface { From af65219531dd729eaa54c2c1dc03a2bc6aa5fd48 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 09:27:43 +0300 Subject: [PATCH 48/57] UHF-8706: better command class name --- .../{HelfiApiCommands.php => GoogleIndexingApiCommands.php} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename public/modules/custom/helfi_google_api/src/Drush/Commands/{HelfiApiCommands.php => GoogleIndexingApiCommands.php} (98%) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php similarity index 98% rename from public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php rename to public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php index ee478731..0ff8f09a 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/HelfiApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php @@ -19,7 +19,7 @@ /** * A Drush command file. */ -final class HelfiApiCommands extends DrushCommands { +final class GoogleIndexingApiCommands extends DrushCommands { use AutowireTrait; From 6965aecb62651f42b84e0da4e6d1f4b5e0ab3969 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 09:47:18 +0300 Subject: [PATCH 49/57] UHF-8706: code fixes --- .../src/EventSubscriber/JobPublishStateSubscriber.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php index 5994e6bf..3710c84a 100644 --- a/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php +++ b/public/modules/custom/helfi_google_api/src/EventSubscriber/JobPublishStateSubscriber.php @@ -11,7 +11,7 @@ use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** - * Subscribe to job publishing events + * Subscribe to job publishing events. */ class JobPublishStateSubscriber implements EventSubscriberInterface { @@ -35,8 +35,6 @@ public static function getSubscribedEvents(): array { SchedulerEvents::PUBLISH_IMMEDIATELY => 'sendIndexRequest', SchedulerEvents::UNPUBLISH => 'sendDeindexingRequest', ]; - - return []; } /** From fc957cf2dd32a719bcc659b65f7247d33d39eab9 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 10:06:45 +0300 Subject: [PATCH 50/57] UHF-8706: function not used any more --- .../custom/helfi_google_api/src/GoogleApi.php | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 1a6658da..82bcd48e 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -47,19 +47,7 @@ public function __construct( private readonly Indexing $indexingService, ) { } - - /** - * Does the api key exist. - * - * @return bool - * Api key exists. - */ - public function hasAuthenticationKey(): bool { - $config = $this->configFactory->get('helfi_google_api.settings'); - $key = $config->get('indexing_api_key') ?: ''; - return (bool) $key; - } - + /** * Correct environment and key is set. * From 7af6cb9c232fe0895f33a1c0ea56c1e9c326c08e Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 10:13:42 +0300 Subject: [PATCH 51/57] UHF-8706: status check debug function to use the dry run logic properly as well --- public/modules/custom/helfi_google_api/src/GoogleApi.php | 7 +++++-- .../custom/helfi_google_api/src/JobIndexingService.php | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 82bcd48e..3b0d6236 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -47,7 +47,7 @@ public function __construct( private readonly Indexing $indexingService, ) { } - + /** * Correct environment and key is set. * @@ -113,6 +113,7 @@ public function indexBatch(array $urls, bool $update): Response { * Request url indexing status. * * Returns the dates of last update and delete requests. + * For debugging purposes only, since it spends the quota. * * @param string $url * The url which indexing status you want to request. @@ -125,9 +126,11 @@ public function checkIndexingStatus(string $url): string { $client->setUseBatch(FALSE); if ($this->isDryRun()) { - $client = $client->authorize(); + return "Dry running index status query with url: $url"; } + $client = $client->authorize(); + $baseUrl = self::METADATA_ENDPOINT; $query_parameter = '?url=' . urlencode($url); $theUrl = $baseUrl . $query_parameter; diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 33ec0b3b..897b5fd5 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -144,6 +144,8 @@ public function deindexEntity(JobListing $entity): Response { /** * Check url indexing status. * + * Status check request uses the api quota. + * * @param string $url * An url to check. * @@ -157,6 +159,8 @@ public function checkItemIndexStatus(string $url): string { /** * If entity seems to be indexed, send a status query. * + * Status check request uses the api quota. + * * @param \Drupal\helfi_rekry_content\Entity\JobListing $entity * Entity to check. * From 07e5165b8bf92246450a8c1c8579427999377ec5 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 10:34:03 +0300 Subject: [PATCH 52/57] UHF-8706: request debug to dryrun, fixed dryrun logic and fixed comments. added debug message logging for dryruns --- .../Commands/GoogleIndexingApiCommands.php | 2 +- .../src/GoogleServiceFactory.php | 1 - .../src/JobIndexingService.php | 26 ++++++++++++++----- .../custom/helfi_google_api/src/Response.php | 8 +++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php b/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php index 0ff8f09a..6edb6576 100644 --- a/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php +++ b/public/modules/custom/helfi_google_api/src/Drush/Commands/GoogleIndexingApiCommands.php @@ -189,7 +189,7 @@ private function handleResponse(Response $response): int { return DrushCommands::EXIT_FAILURE_WITH_CLARITY; } - if ($response->isDebug()) { + if ($response->isDryRun()) { $urls = $response->getUrls(); $this->io()->writeln('The api request would have sent following data: ' . json_encode($urls)); return DrushCommands::EXIT_SUCCESS; diff --git a/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php b/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php index 790fab3f..fd9e00ef 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php +++ b/public/modules/custom/helfi_google_api/src/GoogleServiceFactory.php @@ -36,7 +36,6 @@ public function create(ConfigFactoryInterface $configFactory): Indexing { $client->addScope(self::SCOPES); $client->setUseBatch(TRUE); - // Set empty json object as a key if necessary. if ($key) { $client->setAuthConfig(json_decode($key, TRUE)); $client->authorize(); diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 897b5fd5..c36f2f0a 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -41,7 +41,9 @@ public function __construct( */ public function handleIndexingRequest(array $urls, bool $update): Response { try { - return $this->googleApi->indexBatch($urls, $update); + $response = $this->googleApi->indexBatch($urls, $update); + $this->handleDebugMessage($response); + return $response; } catch (GuzzleException $e) { $message = "Request failed with code {$e->getCode()}: {$e->getMessage()}"; @@ -127,7 +129,7 @@ public function deindexEntity(JobListing $entity): Response { } // No need to delete redirects on debug run. - if (!$result->isDebug()) { + if (!$result->isDryRun()) { $redirect->delete(); } @@ -277,8 +279,8 @@ public function createTemporaryRedirectUrl(JobListing $entity, string $langcode) 'status_code' => 301, ]); - // If the api is not set up, no need to create the redirect. - if ($this->googleApi->isDryRun()) { + // Only save the redirect if module set up properly. + if (!$this->googleApi->isDryRun()) { $redirect->save(); } @@ -309,8 +311,8 @@ public function getExistingTemporaryRedirect(JobListing $entity, string $langcod ->execute(); $redirects = Redirect::loadMultiple($redirectIds); - // For debugging purposes, dbugging won't save the redirect. - if (!$redirects && !$this->googleApi->isDryRun()) { + // For debugging purposes, debugging won't save the redirect. + if (!$redirects && $this->googleApi->isDryRun()) { return $this->createTemporaryRedirectUrl($entity, $langcode)['redirect']; } @@ -343,4 +345,16 @@ private function getEntityAlias(JobListing $entity, string $langcode): string { return $this->aliasManager->getAliasByPath("/node/{$entity->id()}", $langcode); } + /** + * Send debug message if in debug mode. + * + * @param Response $response + * The response. + */ + private function handleDebugMessage(Response $response) { + if ($response->isDryRun()) { + $this->logger->debug('Request would have sent following urls to api: '. json_encode($response->getUrls())); + } + } + } diff --git a/public/modules/custom/helfi_google_api/src/Response.php b/public/modules/custom/helfi_google_api/src/Response.php index 4b98b7a6..648c595d 100644 --- a/public/modules/custom/helfi_google_api/src/Response.php +++ b/public/modules/custom/helfi_google_api/src/Response.php @@ -16,13 +16,13 @@ class Response { * Url which were indexed. * @param array $errors * Errors per url. - * @param bool $debug + * @param bool $dryRun * The request was not sent. */ public function __construct( private array $urls, private array $errors = [], - private bool $debug = FALSE, + private bool $dryRun = FALSE, ) { } @@ -55,8 +55,8 @@ public function getErrors(): array { * @return bool * This is a debug run. */ - public function isDebug(): bool { - return $this->debug; + public function isDryRun(): bool { + return $this->dryRun; } } From 4249772e2a5c9fb5d563f6e0148da6060969f83b Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 10:36:19 +0300 Subject: [PATCH 53/57] UHF-8706: feature is disabled by default in production --- public/sites/default/production.settings.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index 6cde48f0..a059086d 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -3,4 +3,5 @@ $config['openid_connect.client.tunnistamo']['settings']['is_production'] = TRUE; $config['helfi_proxy.settings']['tunnistamo_return_url'] = '/fi/avoimet-tyopaikat/openid-connect/tunnistamo'; $config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE_INDEXING_API_KEY'); -$config['helfi_google_api.settings']['enabled'] = getenv('APP_ENV') === 'production'; +// Remove the comment when it's time to enable the feature on production +// $config['helfi_google_api.settings']['enabled'] = getenv('APP_ENV') === 'production'; From b635b4f7da577613fd0036263bd97fdb5acb27c0 Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 10:39:25 +0300 Subject: [PATCH 54/57] UHF-8706: concat requires space --- .../modules/custom/helfi_google_api/src/JobIndexingService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index c36f2f0a..8b1a2b07 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -353,7 +353,7 @@ private function getEntityAlias(JobListing $entity, string $langcode): string { */ private function handleDebugMessage(Response $response) { if ($response->isDryRun()) { - $this->logger->debug('Request would have sent following urls to api: '. json_encode($response->getUrls())); + $this->logger->debug('Request would have sent following urls to api: ' . json_encode($response->getUrls())); } } From 4f329c989b443f3a48a7dfe282954c09ea58a51e Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 11:03:44 +0300 Subject: [PATCH 55/57] UHF-8706: the is enabled check was wrong way --- public/modules/custom/helfi_google_api/src/GoogleApi.php | 8 ++++---- .../custom/helfi_google_api/src/JobIndexingService.php | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 3b0d6236..1904ae79 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -49,7 +49,7 @@ public function __construct( } /** - * Correct environment and key is set. + * Correct environment and key is set and enabled is true. * * @return bool * The api is set up @@ -59,7 +59,7 @@ public function isDryRun(): bool { $key = $config->get('indexing_api_key') ?: ''; $isEnabled = $config->get('enabled') ?: FALSE; - return $key && $isEnabled; + return !$key && !$isEnabled; } /** @@ -74,8 +74,8 @@ public function isDryRun(): bool { * Object which holds the handled urls and request errors. */ public function indexBatch(array $urls, bool $update): Response { - if (!$this->isDryRun()) { - return new Response($urls, debug: TRUE); + if ($this->isDryRun()) { + return new Response($urls, dryRun: TRUE); } $batch = $this->indexingService->createBatch(); diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 8b1a2b07..490635d3 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -228,6 +228,11 @@ public function checkEntityIndexStatus(JobListing $entity): string { * Has temporary redirect. */ public function hasTemporaryRedirect(JobListing $entity, string $langcode): bool { + // In case of dry run, we can always say FALSE. + if ($this->googleApi->isDryRun()) { + return FALSE; + } + $job_alias = $this->getEntityAlias($entity, $langcode); $query = $this->entityTypeManager->getStorage('redirect') From 9a34eeb490162cfa6494c3d765890f375c0fc05d Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 11:34:00 +0300 Subject: [PATCH 56/57] UHF-8706: change enabled to dry_run --- conf/cmi/helfi_google_api.settings.yml | 2 +- public/modules/custom/helfi_google_api/src/GoogleApi.php | 4 ++-- public/sites/default/production.settings.php | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conf/cmi/helfi_google_api.settings.yml b/conf/cmi/helfi_google_api.settings.yml index db0e4c95..f070b2dd 100644 --- a/conf/cmi/helfi_google_api.settings.yml +++ b/conf/cmi/helfi_google_api.settings.yml @@ -1,2 +1,2 @@ -enabled: false +dry_run: true indexing_api_key: '' diff --git a/public/modules/custom/helfi_google_api/src/GoogleApi.php b/public/modules/custom/helfi_google_api/src/GoogleApi.php index 1904ae79..bbcd2534 100644 --- a/public/modules/custom/helfi_google_api/src/GoogleApi.php +++ b/public/modules/custom/helfi_google_api/src/GoogleApi.php @@ -57,9 +57,9 @@ public function __construct( public function isDryRun(): bool { $config = $this->configFactory->get('helfi_google_api.settings'); $key = $config->get('indexing_api_key') ?: ''; - $isEnabled = $config->get('enabled') ?: FALSE; + $dryRun = $config->get('dry_run') ?: TRUE; - return !$key && !$isEnabled; + return !$key || $dryRun; } /** diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index a059086d..cbc42561 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -3,5 +3,6 @@ $config['openid_connect.client.tunnistamo']['settings']['is_production'] = TRUE; $config['helfi_proxy.settings']['tunnistamo_return_url'] = '/fi/avoimet-tyopaikat/openid-connect/tunnistamo'; $config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE_INDEXING_API_KEY'); + // Remove the comment when it's time to enable the feature on production -// $config['helfi_google_api.settings']['enabled'] = getenv('APP_ENV') === 'production'; +// $config['helfi_google_api.settings']['dry_run'] = TRUE; From 697ca1383faa2b181ca89404856a3f45aa5b25cd Mon Sep 17 00:00:00 2001 From: rpnykanen Date: Wed, 11 Sep 2024 11:45:08 +0300 Subject: [PATCH 57/57] UHF-8706: added return type --- .../modules/custom/helfi_google_api/src/JobIndexingService.php | 2 +- public/sites/default/production.settings.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/modules/custom/helfi_google_api/src/JobIndexingService.php b/public/modules/custom/helfi_google_api/src/JobIndexingService.php index 490635d3..99bf0bf6 100644 --- a/public/modules/custom/helfi_google_api/src/JobIndexingService.php +++ b/public/modules/custom/helfi_google_api/src/JobIndexingService.php @@ -356,7 +356,7 @@ private function getEntityAlias(JobListing $entity, string $langcode): string { * @param Response $response * The response. */ - private function handleDebugMessage(Response $response) { + private function handleDebugMessage(Response $response): void { if ($response->isDryRun()) { $this->logger->debug('Request would have sent following urls to api: ' . json_encode($response->getUrls())); } diff --git a/public/sites/default/production.settings.php b/public/sites/default/production.settings.php index cbc42561..ef524b4f 100644 --- a/public/sites/default/production.settings.php +++ b/public/sites/default/production.settings.php @@ -5,4 +5,4 @@ $config['helfi_google_api.settings']['indexing_api_key'] = getenv('GOOGLE_INDEXING_API_KEY'); // Remove the comment when it's time to enable the feature on production -// $config['helfi_google_api.settings']['dry_run'] = TRUE; +// $config['helfi_google_api.settings']['dry_run'] = FALSE;