UHF-9937: Make sure removed job listings are not published

README.md

@@ -8,7 +8,7 @@ search found on the site and provides other recruitment information for the city
 Env | Branch | Drush alias | URL
 --- | ------ | ----------- | ---
 development | * | - | http://helfi-rekry.docker.so/
-production | main | @main | TBD
+production | main | @main | https://www.hel.fi/fi/avoimet-tyopaikat
 
 ## Requirements
 
@@ -110,7 +110,7 @@ listings are published for indexing.
 
 - The `helfi_rekry_content` module code can be found from [here](https://github.com/City-of-Helsinki/drupal-helfi-rekry/tree/dev/public/modules/custom/helfi_rekry_content).
 - The migration interval for the new job listings can be checked from the cron configuration [here](https://github.com/City-of-Helsinki/drupal-helfi-rekry/blob/dev/docker/openshift/crons/migrate-job-listings.sh).
-- The migration interval for changed job listings poll is written on this cron configuration [here](https://github.com/City-of-Helsinki/drupal-helfi-rekry/blob/dev/docker/openshift/crons/migrate-changed-job-listings.sh).
+- The migration interval for images, taxonomy terms, etc. is written on this cron configuration [here](https://github.com/City-of-Helsinki/drupal-helfi-rekry/blob/dev/docker/openshift/crons/migrate-helbit-data.sh).
 - The scheduled publishing interval can be checked from this cron configuration [here](https://github.com/City-of-Helsinki/drupal-helfi-rekry/blob/dev/docker/openshift/crons/content-scheduler.sh).
 
 ### Testing on local

docker/openshift/crons/migrate-helbit-data.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+migrations=(
+  "helfi_rekry_images"
+  "helfi_rekry_videos"
+  "helfi_rekry_task_areas:fi"
+  "helfi_rekry_task_areas:sv"
+  "helfi_rekry_task_areas:en"
+  "helfi_rekry_organizations:fi"
+  "helfi_rekry_organizations:sv"
+  "helfi_rekry_organizations:en"
+  "helfi_rekry_employments"
+  "helfi_rekry_employment_types"
+)
+
+echo "Starting helbit migrations: $(date)"
+while true
+do
+  echo "Running helbit migrations: $(date)"
+
+  for migration in "${migrations[@]}"; do
+    # Allow migrations to be run every 1.5 hours, reset stuck migrations every 12 hours.
+    drush migrate:import "$migration" --reset-threshold 43200 --interval 5400 --no-progress
+  done
+
+  drush helfi-rekry-content:clean-expired-listings
+
+  # Sleep for 3 hours.
+  sleep 10800
+done

docker/openshift/crons/migrate-job-listings.sh

@@ -1,17 +1,7 @@
 #!/bin/bash
 
 migrations=(
-  "helfi_rekry_images:all"
-  "helfi_rekry_videos:all"
-  "helfi_rekry_task_areas:fi"
-  "helfi_rekry_task_areas:sv"
-  "helfi_rekry_task_areas:en"
-  "helfi_rekry_organizations:fi"
-  "helfi_rekry_organizations:sv"
-  "helfi_rekry_organizations:en"
-  "helfi_rekry_employments:all"
-  "helfi_rekry_employment_types:all"
-  "helfi_rekry_jobs:all"
+  "helfi_rekry_jobs"
 )
 
 echo "Starting job listing migrations: $(date)"
@@ -20,12 +10,10 @@ do
   echo "Running job listing migrations: $(date)"
 
   for migration in "${migrations[@]}"; do
-    # Allow migrations to be run every 1.5 hours, reset stuck migrations every 12 hours.
-    drush migrate:import "$migration" --reset-threshold 43200 --interval 5400 --no-progress
+    # Allow migrations to be run every 10 minutes, reset stuck migrations every 30 minutes.
+    drush migrate:import "$migration" --reset-threshold 1800 --interval 600 --no-progress
   done
 
-  drush helfi-rekry-content:clean-expired-listings
-
-  # Sleep for 3 hours.
-  sleep 10800
+  # Sleep for 15 minutes.
+  sleep 900
 done

public/modules/custom/helfi_rekry_content/migrations/employment_types.yml

@@ -5,7 +5,6 @@ dependencies:
     module:
       - helfi_rekry_content
 id: helfi_rekry_employment_types
-deriver: Drupal\helfi_rekry_content\Plugin\Deriver\HelbitMigrationDeriver
 migration_tags:
   - helfi_rekry_content
   - helfi_rekry_jobs

public/modules/custom/helfi_rekry_content/migrations/employments.yml

@@ -5,7 +5,6 @@ dependencies:
     module:
       - helfi_rekry_content
 id: helfi_rekry_employments
-deriver: Drupal\helfi_rekry_content\Plugin\Deriver\HelbitMigrationDeriver
 migration_tags:
   - helfi_rekry_content
   - helfi_rekry_jobs

public/modules/custom/helfi_rekry_content/migrations/job_listing_images.yml

@@ -5,7 +5,6 @@ dependencies:
     module:
       - helfi_rekry_content
 id: helfi_rekry_images
-deriver: Drupal\helfi_rekry_content\Plugin\Deriver\HelbitMigrationDeriver
 migration_tags:
   - helfi_rekry_content
   - helfi_rekry_jobs

public/modules/custom/helfi_rekry_content/migrations/job_listing_videos.yml

@@ -5,7 +5,6 @@ dependencies:
     module:
       - helfi_rekry_content
 id: helfi_rekry_videos
-deriver: Drupal\helfi_rekry_content\Plugin\Deriver\HelbitMigrationDeriver
 migration_tags:
   - helfi_rekry_content
   - helfi_rekry_jobs

public/modules/custom/helfi_rekry_content/migrations/job_listings.yml

@@ -5,7 +5,6 @@ dependencies:
     module:
       - helfi_rekry_content
 id: helfi_rekry_jobs
-deriver: Drupal\helfi_rekry_content\Plugin\Deriver\HelbitMigrationDeriver
 migration_tags:
   - helfi_rekry_content
   - helfi_rekry_jobs

public/modules/custom/helfi_rekry_content/src/EventSubscriber/JobListingHideMissingSubscriber.php

@@ -7,6 +7,7 @@
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Queue\QueueFactory;
 use Drupal\Core\StringTranslation\StringTranslationTrait;
+use Drupal\node\NodeInterface;
 use Drush\Drupal\Migrate\MigrateMissingSourceRowsEvent;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\DependencyInjection\Attribute\Autowire;
@@ -83,14 +84,21 @@ function ($values, $destinationValue) {
       return;
     }
 
-    // Query missing nodes that are still published.
-    $query = $this->entityTypeManager->getStorage('node')
-      ->getQuery()
+    $query = $this->entityTypeManager
+      ->getStorage('node')
+      ->getQuery();
+
+    // Query missing nodes that are still published
+    // or still scheduled to be published.
+    $orCondition = $query->orConditionGroup();
+    $orCondition->condition('status', NodeInterface::PUBLISHED);
+    $orCondition->exists('publish_on');
+
+    $nids = $query
       ->accessCheck(FALSE)
       ->condition('nid', $destinationIds, 'IN')
-      ->condition('status', 1)
-      ->notExists('unpublish_on');
-    $nids = $query->execute();
+      ->condition($orCondition)
+      ->execute();
 
     foreach ($nids as $nid) {
       $job = ['nid' => $nid];

public/modules/custom/helfi_rekry_content/src/Plugin/Deriver/HelbitMigrationDeriver.php

@@ -36,41 +36,14 @@ public static function create(ContainerInterface $container, $base_plugin_id) :
    */
   public function getDerivativeDefinitions($base_plugin_definition): array {
     $langcodes = ['fi', 'sv', 'en'];
-    $derivatives = ['all', 'changed'];
 
-    if ($base_plugin_definition['source']['plugin'] == 'helbit_open_jobs') {
-      foreach ($derivatives as $key) {
-        $this->derivatives[$key] = $this->getJobMigrationDerivativeValues($base_plugin_definition, $key);
-      }
-    }
-    else {
-      foreach ($langcodes as $langcode) {
-        $this->derivatives[$langcode] = $this->getDerivativeValues($base_plugin_definition, $langcode);
-      }
+    foreach ($langcodes as $langcode) {
+      $this->derivatives[$langcode] = $this->getDerivativeValues($base_plugin_definition, $langcode);
     }
 
     return $this->derivatives;
   }
 
-  /**
-   * Creates a derivative definition for job migration.
-   *
-   * @param array $base_plugin_definition
-   *   Base migration definitions.
-   * @param string $key
-   *   Key for derivative.
-   *
-   * @return array
-   *   Modified plugin definition for derivative.
-   */
-  private function getJobMigrationDerivativeValues(array $base_plugin_definition, string $key): array {
-    if (str_starts_with($key, 'changed')) {
-      $base_plugin_definition['source']['changed'] = TRUE;
-    }
-
-    return $base_plugin_definition;
-  }
-
   /**
    * Creates a derivative definition for each available language.
    *

public/modules/custom/helfi_rekry_content/src/Plugin/QueueWorker/UnpublishWorker.php

@@ -6,20 +6,21 @@
 
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\Core\Queue\Attribute\QueueWorker;
 use Drupal\Core\Queue\QueueWorkerBase;
+use Drupal\Core\StringTranslation\TranslatableMarkup;
 use Drupal\node\NodeInterface;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * Queue Worker for removing job listings not present in source.
- *
- * @QueueWorker(
- *   id = "job_listing_unpublish_worker",
- *   title = @Translation("Job listing unpublish worker"),
- *   cron = {"time" = 60}
- * )
  */
+#[QueueWorker(
+  id: 'job_listing_unpublish_worker',
+  title: new TranslatableMarkup('Job listing unpublish worker'),
+  cron: ['time' => 60],
+)]
 final class UnpublishWorker extends QueueWorkerBase implements ContainerFactoryPluginInterface {
 
   /**
@@ -74,23 +75,25 @@ public function processItem($data): void {
 
     // Unpublish all translations.
     if ($node instanceof NodeInterface && $node->getType() == 'job_listing') {
-      foreach (['fi', 'sv', 'en'] as $langcode) {
+      foreach ($node->getTranslationLanguages() as $language) {
+        $langcode = $language->getId();
+
         // Unpublish the job listing node as it's still published, but it's
         // no longer available at the source.
         if (!$node->hasTranslation($langcode)) {
           continue;
         }
 
         $translation = $node->getTranslation($langcode);
-        if ($translation->isPublished()) {
-          $translation->setUnpublished();
-          if ($translation->hasField('publish_on') && !empty($translation->get('publish_on')->getValue())) {
-            // Also clear the publish on date to make sure the translation is
-            // not going to be re-published.
-            $translation->set('publish_on', NULL);
-          }
-          $translation->save();
+        $translation->setUnpublished();
+
+        // Also clear the published on date so the translation
+        // is not going to be re-published.
+        if ($translation->hasField('publish_on') && !empty($translation->get('publish_on')->getValue())) {
+          $translation->set('publish_on', NULL);
         }
+
+        $translation->save();
       }
     }
 

public/modules/custom/helfi_rekry_content/src/Plugin/migrate/source/HelbitOpenJobs.php

@@ -50,10 +50,6 @@ protected function initializeIterator(): \Iterator {
 
     $query = [];
 
-    if ($this->configuration['changed'] ?? FALSE) {
-      $query['timestamp'] = date('Y-m-d\TH:m:i', strtotime('-1 day'));
-    }
-
     foreach ($langcodes as $langcode) {
       foreach ($this->helbit->getJobListings($langcode, $query) as $row) {
         $fields = $this->getFieldsFromRow($row) + [

public/modules/custom/helfi_rekry_content/tests/src/Kernel/HelbitMigrationDeriverTest.php

@@ -21,19 +21,6 @@ class HelbitMigrationDeriverTest extends KernelTestBase {
    * Tests Helbit deriver.
    */
   public function testHelbitDeriver(): void {
-    $deriver = new HelbitMigrationDeriver(new Settings(self::TEST_HELBIT_KEY));
-    $result = $deriver->getDerivativeDefinitions([
-      'id' => 'helfi_rekry_jobs',
-      'source' => [
-        'plugin' => 'helbit_open_jobs',
-      ],
-    ]);
-
-    $this->assertArrayHasKey('all', $result);
-    $this->assertArrayHasKey('changed', $result);
-    $this->assertEmpty($result['all']['source']['changed'] ?? NULL);
-    $this->assertTrue($result['changed']['source']['changed']);
-
     $deriver = new HelbitMigrationDeriver(new Settings(self::TEST_HELBIT_KEY));
     $result = $deriver->getDerivativeDefinitions([
       'id' => 'helfi_rekry_task_areas',

public/modules/custom/helfi_rekry_content/tests/src/Kernel/HelbitOpenJobsTest.php

@@ -50,29 +50,6 @@ public function testException(): void {
       ->next();
   }
 
-  /**
-   * Tests that changed timestamp is present in request when configured.
-   */
-  public function testChangedTimestamp(): void {
-    $isValidTime = static fn (array $query) => preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/', $query['timestamp']);
-
-    $helbit = $this->prophesize(HelbitClient::class);
-    $helbit
-      ->getJobListings(Argument::any(), Argument::that($isValidTime))
-      ->shouldBeCalled();
-
-    $this
-      ->getSut($helbit->reveal(), [
-        'changed' => TRUE,
-        'ids' => [
-          'id' => [
-            'type' => 'string',
-          ],
-        ],
-      ])
-      ->next();
-  }
-
   /**
    * Tests iterator.
    */