Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Keep track when was an item last seen in a feed #914

Merged
merged 1 commit into from
Apr 15, 2017
Merged

Keep track when was an item last seen in a feed #914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions daos/mysql/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,35 @@ public function __construct() {
'INSERT INTO `' . \F3::get('db_prefix') . 'version` (version) VALUES (10);'
]);
}
if (strnatcmp($version, '11') < 0) {
\F3::get('db')->exec([
'DROP TRIGGER insert_updatetime_trigger',
'DROP TRIGGER update_updatetime_trigger',
'ALTER TABLE ' . \F3::get('db_prefix') . 'items ADD lastseen DATETIME',
'UPDATE ' . \F3::get('db_prefix') . 'items SET lastseen = CURRENT_TIMESTAMP',
// Needs to be a trigger since MySQL before 5.6.5 does not support default value for DATETIME.
// https://dev.mysql.com/doc/relnotes/mysql/5.6/en/news-5-6-5.html#mysqld-5-6-5-data-types
// Needs to be a single trigger due to MySQL before 5.7.2 not supporting multiple triggers for the same event on the same table.
// https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-2.html#mysqld-5-7-2-triggers
'CREATE TRIGGER ' . \F3::get('db_prefix') . 'items_insert_trigger
BEFORE INSERT ON ' . \F3::get('db_prefix') . 'items FOR EACH ROW
BEGIN
SET NEW.updatetime = NOW();
SET NEW.lastseen = NOW();
END;',
'CREATE TRIGGER ' . \F3::get('db_prefix') . 'items_update_trigger
BEFORE UPDATE ON ' . \F3::get('db_prefix') . 'items FOR EACH ROW
BEGIN
IF (
OLD.unread <> NEW.unread OR
OLD.starred <> NEW.starred
) THEN
SET NEW.updatetime = NOW();
END IF;
END;',
'INSERT INTO ' . \F3::get('db_prefix') . 'version (version) VALUES (11)'
]);
}
}

// just initialize once
Expand Down
24 changes: 18 additions & 6 deletions daos/mysql/Items.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,12 @@ public function exists($uid) {
}

/**
* search whether given ids are already in database or not
* search whether given uids are already in database or not
*
* @param array $itemsInFeed list with ids for checking whether they are already in database or not
* @param int $sourceId the id of the source to search for the items
*
* @return array with all existing ids from itemsInFeed (array (id => true))
* @return array with all existing uids from itemsInFeed (array (uid => id))
*/
public function findAll($itemsInFeed, $sourceId) {
$itemsFound = [];
Expand All @@ -158,23 +158,35 @@ public function findAll($itemsInFeed, $sourceId) {
array_walk($itemsInFeed, function(&$value) {
$value = \F3::get('db')->quote($value);
});
$query = 'SELECT uid AS uid FROM ' . \F3::get('db_prefix') . 'items WHERE source = ' . \F3::get('db')->quote($sourceId) . ' AND uid IN (' . implode(',', $itemsInFeed) . ')';
$query = 'SELECT id, uid AS uid FROM ' . \F3::get('db_prefix') . 'items WHERE source = ' . \F3::get('db')->quote($sourceId) . ' AND uid IN (' . implode(',', $itemsInFeed) . ')';
$res = \F3::get('db')->query($query);
if ($res) {
$all = $res->fetchAll();
foreach ($all as $row) {
$uid = $row['uid'];
$itemsFound[$uid] = true;
$itemsFound[$uid] = $row['id'];
}
}

return $itemsFound;
}

/**
* Update the time items were last seen in the feed to prevent unwanted cleanup.
*
* @param array $itemIds
*
* @return void
*/
public function updateLastSeen(array $itemIds) {
\F3::get('db')->exec('UPDATE ' . \F3::get('db_prefix') . 'items SET lastseen = CURRENT_TIMESTAMP
WHERE ' . $this->stmt->intRowMatches('id', $itemIds));
}

/**
* cleanup orphaned and old items
*
* @param DateTime $date date to delete all items older than this value [optional]
* @param ?DateTime $date date to delete all items older than this value
*
* @return void
*/
Expand All @@ -184,7 +196,7 @@ public function cleanup(\DateTime $date = null) {
SELECT id FROM ' . \F3::get('db_prefix') . 'sources)');
if ($date !== null) {
\F3::get('db')->exec('DELETE FROM ' . \F3::get('db_prefix') . 'items
WHERE ' . $this->stmt->isFalse('starred') . ' AND datetime<:date',
WHERE ' . $this->stmt->isFalse('starred') . ' AND lastseen<:date',
[':date' => $date->format('Y-m-d') . ' 00:00:00']
);
}
Expand Down
14 changes: 14 additions & 0 deletions daos/pgsql/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,20 @@ public function __construct() {
'INSERT INTO version (version) VALUES (10);'
]);
}
if (strnatcmp($version, '11') < 0) {
\F3::get('db')->exec([
'DROP TRIGGER update_updatetime_trigger ON items',
'ALTER TABLE items ADD lastseen TIMESTAMP(0) WITH TIME ZONE NOT NULL DEFAULT NOW()',
'CREATE TRIGGER update_updatetime_trigger
BEFORE UPDATE ON items FOR EACH ROW
WHEN (
OLD.unread IS DISTINCT FROM NEW.unread OR
OLD.starred IS DISTINCT FROM NEW.starred
)
EXECUTE PROCEDURE update_updatetime_procedure();',
'INSERT INTO version (version) VALUES (11);'
]);
}
}

// just initialize once
Expand Down
39 changes: 39 additions & 0 deletions daos/sqlite/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,45 @@ public function __construct() {
INSERT INTO version (version) VALUES (9);
');
}
if (strnatcmp($version, '11') < 0) {
\F3::get('db')->exec([
// Table needs to be re-created because ALTER TABLE is rather limited.
// https://sqlite.org/lang_altertable.html#otheralter
'CREATE TABLE new_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
datetime DATETIME NOT NULL,
title TEXT NOT NULL,
content TEXT NOT NULL,
thumbnail TEXT,
icon TEXT,
unread BOOL NOT NULL,
starred BOOL NOT NULL,
source INT NOT NULL,
uid VARCHAR(255) NOT NULL,
link TEXT NOT NULL,
updatetime DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
author VARCHAR(255),
shared BOOL,
lastseen DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
)',
'INSERT INTO new_items SELECT *, CURRENT_TIMESTAMP FROM items',
'DROP TABLE items',
'ALTER TABLE new_items RENAME TO items',
'CREATE INDEX source ON items (source)',
'CREATE TRIGGER update_updatetime_trigger
AFTER UPDATE ON items FOR EACH ROW
WHEN (
OLD.unread <> NEW.unread OR
OLD.starred <> NEW.starred
)
BEGIN
UPDATE items
SET updatetime = CURRENT_TIMESTAMP
WHERE id = NEW.id;
END',
'INSERT INTO version (version) VALUES (11)'
]);
}
}

// just initialize once
Expand Down
8 changes: 8 additions & 0 deletions helpers/ContentLoader.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,12 @@ public function fetch($source) {
$itemsFound = $this->itemsDao->findAll($itemsInFeed, $source['id']);

$lasticon = false;
$itemsSeen = [];
foreach ($spout as $item) {
// item already in database?
if (isset($itemsFound[$item->getId()])) {
\F3::get('logger')->debug('item "' . $item->getTitle() . '" already in database.');
$itemsSeen[] = $itemsFound[$item->getId()];
continue;
}

Expand Down Expand Up @@ -211,6 +214,11 @@ public function fetch($source) {

// remove previous errors and set last update timestamp
$this->updateSource($source, $lastEntry);

// mark items seen in the feed to prevent premature garbage removal
if (count($itemsSeen) > 0) {
$this->itemsDao->updateLastSeen($itemsSeen);
}
}

/**
Expand Down