Adds migrations for page events columns

postgres/client.js

@@ -5,7 +5,7 @@ const { POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES
   { parseOrNot, wrapInObject, decode } = require('../services/utils'),
   { findSchemaAndTable, wrapJSONStringInObject } = require('../services/utils'),
   knexLib = require('knex'),
-  { isList, isUri } = require('clayutils'),
+  { isList, isUri, isPage } = require('clayutils'),
   TransformStream = require('../services/list-transform-stream'),
   META_PUT_PATCH_FN = patch('meta');
 var knex, log = require('../services/log').setup({ file: __filename });
@@ -272,16 +272,52 @@ function getMeta(key) {
  * @param {Object} value [description]
  * @return {Promise} [description]
  */
+/* eslint complexity: 0 */
 function putMeta(key, value) {
   const { schema, table } = findSchemaAndTable(key),
-    map = columnToValueMap('id', key);
+    map = columnToValueMap('id', key),
+    parsedValue = parseOrNot(value);
 
   // add meta column to map
-  columnToValueMap('meta', parseOrNot(value), map);
+  columnToValueMap('meta', parsedValue, map);
+
+  if (isPage(key)) {
+    // If value has first publish time, add it to the map
+    if (parsedValue.firstPublishTime)
+      columnToValueMap('published_at', parsedValue.firstPublishTime, map);
+
+    // If value has publish time, add it as republished date
+    if (parsedValue.publishTime)
+      columnToValueMap('republished_at', parsedValue.publishTime, map);
+
+    // If we have history data, then find the unpublish and archive events
+    if (parsedValue.history && parsedValue.history.length) {
+      const latestUnpublish = getLatestActionByName(parsedValue.history, 'unpublish'),
+        latestArchived = getLatestActionByName(parsedValue.history, 'archive');
+
+      if (latestUnpublish.timestamp)
+        columnToValueMap('unpublished_at', latestUnpublish.timestamp, map);
+
+      if (latestArchived.timestamp)
+        columnToValueMap('archived_at', latestArchived.timestamp, map);
+    }
+  }
 
   return onConflictPut(map, schema, table).then(() => map.meta);
 }
 
+/**
+ * Gets the latest entry in the history based on the action
+ * @param {Object[]} history
+ * @param {String} action
+ * @returns {Object}
+ */
+function getLatestActionByName(history, action) {
+  return history
+    .filter(event => event.action === action)
+    .pop() || {};
+}
+
 /**
  * Creates a table with the name that's
  * passed into the function. Table has

postgres/client.test.js

@@ -605,6 +605,57 @@ describe('postgres/client', () => {
         expect(data).toEqual(data);
       });
     });
+
+    test('if uri is page, insert publication events dates', () => {
+      const key = 'nymag.com/_pages/sample-page',
+        meta = {
+          someText: '',
+          someOtherText: '',
+          history: [{
+            action: 'unpublish',
+            timestamp: '2019-06-14'
+          }, {
+            action: 'archive',
+            timestamp: '2019-06-14'
+          }]
+        };
+
+      return client.putMeta(key, meta).then((data) => {
+        expect(data).toEqual(meta);
+      });
+    });
+
+    test('if uri is page and does not have history, insert without publication dates', () => {
+      const key = 'nymag.com/_pages/sample-page',
+        meta = {
+          someText: '',
+          someOtherText: '',
+          firstPublishTime: '2019-06-14',
+          publishTime: '2019-06-14'
+        };
+
+      return client.putMeta(key, meta).then((data) => {
+        expect(data).toEqual(meta);
+      });
+    });
+
+    test('if uri is page, but history does not have unpublish/archive events, insert without those dates', () => {
+      const key = 'nymag.com/_pages/sample-page',
+        meta = {
+          someText: '',
+          someOtherText: '',
+          firstPublishTime: '2019-06-14',
+          publishTime: '2019-06-14',
+          history: [{
+            action: 'create',
+            timestamp: '2019-06-14'
+          }]
+        };
+
+      return client.putMeta(key, meta).then((data) => {
+        expect(data).toEqual(meta);
+      });
+    });
   });
 
   describe('batch', () => {

postgres/index.js

@@ -42,6 +42,7 @@ function createTables() {
   return bluebird.all(getComponents().map(component => client.createTable(`components.${component}`)))
     .then(() => bluebird.all(getLayouts().map(layout => client.createTableWithMeta(`layouts.${layout}`))))
     .then(() => client.createTableWithMeta('pages'))
+    .then(() => client.raw('CREATE TABLE IF NOT EXISTS ?? ( id TEXT PRIMARY KEY NOT NULL, data TEXT NOT NULL, url TEXT );', ['uris']))
     .then(() => createRemainingTables());
 }
 
@@ -59,24 +60,22 @@ function setup(testPostgresHost) {
   }
 
   return client.connect()
-    .then(() => {
-      return migrate(
-        {
-          database: POSTGRES_DB,
-          user: POSTGRES_USER,
-          password: POSTGRES_PASSWORD,
-          host: postgresHost,
-          port: POSTGRES_PORT
-        },
-        path.join(__dirname, '../services/migrations')
-      );
-    })
-    .then(() => {
-      log('info', 'Migrations Complete');
-    })
-    .then(() => createTables())
+    .then(() => client.createSchema('components'))
+    .then(() => client.createSchema('layouts'))
+    .then(createTables)
+    .then(() => migrate(
+      {
+        database: POSTGRES_DB,
+        user: POSTGRES_USER,
+        password: POSTGRES_PASSWORD,
+        host: postgresHost,
+        port: POSTGRES_PORT
+      },
+      path.join(__dirname, '../services/migrations')
+    ))
+    .then(() => log('info', 'Migrations Complete'))
     .then(() => ({ server: `${postgresHost}:${POSTGRES_PORT}` }))
-    .catch(logGenericError);
+    .catch(logGenericError(__filename));
 }
 
 module.exports.setup = setup;

services/migrations/001_add_publish_event_columns.sql

@@ -0,0 +1,5 @@
+ALTER TABLE IF EXISTS pages
+ADD COLUMN IF NOT EXISTS published_at TIMESTAMPTZ,
+ADD COLUMN IF NOT EXISTS unpublished_at TIMESTAMPTZ,
+ADD COLUMN IF NOT EXISTS republished_at TIMESTAMPTZ,
+ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;

services/migrations/002_set_unpublished_at_values.sql

@@ -0,0 +1,11 @@
+UPDATE pages
+SET unpublished_at = subquery.unpublish_date
+FROM (SELECT DISTINCT ON (id) id,
+  events ->> 'action' AS action,
+  TO_TIMESTAMP(events ->> 'timestamp', 'YYYY-MM-DD HH24:MI:SSZ') AS unpublish_date
+  FROM pages AS p,
+    JSONB_ARRAY_ELEMENTS(p.meta -> 'history') AS events
+  WHERE meta -> 'history' IS NOT NULL
+    AND events ->> 'action' = 'unpublish'
+  ORDER BY id, unpublish_date DESC) AS subquery
+WHERE pages.id = subquery.id;

services/migrations/003_set_archived_at_values.sql

@@ -0,0 +1,11 @@
+UPDATE pages
+SET archived_at = subquery.archive_date
+FROM (SELECT DISTINCT ON (id) id,
+  events ->> 'action' AS action,
+  TO_TIMESTAMP(events ->> 'timestamp', 'YYYY-MM-DD HH24:MI:SSZ') AS archive_date
+  FROM pages AS p,
+    JSONB_ARRAY_ELEMENTS(p.meta -> 'history') AS events
+  WHERE meta -> 'history' IS NOT NULL
+    AND events ->> 'action' = 'archive'
+  ORDER BY id, archive_date DESC) AS subquery
+WHERE pages.id = subquery.id;

services/migrations/004_set_republished_at_values.sql

@@ -0,0 +1,7 @@
+UPDATE pages
+SET republished_at = subquery.republish_time
+FROM (SELECT id, TO_TIMESTAMP(meta ->> 'publishTime', 'YYYY-MM-DD HH24:MI:SSZ') AS republish_time
+  FROM pages
+  WHERE meta IS NOT NULL
+  AND meta ->> 'publishTime' IS NOT NULL) AS subquery
+WHERE pages.id = subquery.id;

services/migrations/005_set_published_at_values.sql

@@ -0,0 +1,7 @@
+UPDATE pages
+SET published_at = subquery.first_publish_time
+FROM (SELECT id, TO_TIMESTAMP(meta ->> 'firstPublishTime', 'YYYY-MM-DD HH24:MI:SSZ') as first_publish_time
+  FROM pages
+  WHERE meta IS NOT NULL
+  AND meta ->> 'firstPublishTime' IS NOT NULL) AS subquery
+WHERE pages.id = subquery.id;