From c1bc8f6c827a27540b5510b42dcc65b39d38f2c1 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <alec@alec.pl>
Date: Mon, 14 Oct 2013 07:19:37 -0400
Subject: [PATCH] Change so abort=true does not break the loop in exec_hook(), provide a new 'break' flag for this purpose
---
program/lib/Roundcube/rcube_imap_cache.php | 369 +++++++++++++++++++++++++++++++++++++---------------
1 files changed, 259 insertions(+), 110 deletions(-)
diff --git a/program/lib/Roundcube/rcube_imap_cache.php b/program/lib/Roundcube/rcube_imap_cache.php
index f33ac07..445d463 100644
--- a/program/lib/Roundcube/rcube_imap_cache.php
+++ b/program/lib/Roundcube/rcube_imap_cache.php
@@ -27,6 +27,9 @@
*/
class rcube_imap_cache
{
+ const MODE_INDEX = 1;
+ const MODE_MESSAGE = 2;
+
/**
* Instance of rcube_imap
*
@@ -49,6 +52,20 @@
private $userid;
/**
+ * Expiration time in seconds
+ *
+ * @var int
+ */
+ private $ttl;
+
+ /**
+ * Maximum cached message size
+ *
+ * @var int
+ */
+ private $threshold;
+
+ /**
* Internal (in-memory) cache
*
* @var array
@@ -56,6 +73,7 @@
private $icache = array();
private $skip_deleted = false;
+ private $mode;
/**
* List of known flags. Thanks to this we can handle flag changes
@@ -81,15 +99,32 @@
);
+
/**
* Object constructor.
+ *
+ * @param rcube_db $db DB handler
+ * @param rcube_imap $imap IMAP handler
+ * @param int $userid User identifier
+ * @param bool $skip_deleted skip_deleted flag
+ * @param string $ttl Expiration time of memcache/apc items
+ * @param int $threshold Maximum cached message size
*/
- function __construct($db, $imap, $userid, $skip_deleted)
+ function __construct($db, $imap, $userid, $skip_deleted, $ttl=0, $threshold=0)
{
+ // convert ttl string to seconds
+ $ttl = get_offset_sec($ttl);
+ if ($ttl > 2592000) $ttl = 2592000;
+
$this->db = $db;
$this->imap = $imap;
$this->userid = $userid;
$this->skip_deleted = $skip_deleted;
+ $this->ttl = $ttl;
+ $this->threshold = $threshold;
+
+ // cache all possible information by default
+ $this->mode = self::MODE_INDEX | self::MODE_MESSAGE;
}
@@ -100,6 +135,17 @@
{
$this->save_icache();
$this->icache = null;
+ }
+
+
+ /**
+ * Set cache mode
+ *
+ * @param int $mode Cache mode
+ */
+ public function set_mode($mode)
+ {
+ $this->mode = $mode;
}
@@ -215,9 +261,7 @@
* Return messages thread.
* If threaded index doesn't exist or is invalid, will be updated.
*
- * @param string $mailbox Folder name
- * @param string $sort_field Sorting column
- * @param string $sort_order Sorting order (ASC|DESC)
+ * @param string $mailbox Folder name
*
* @return array Messages threaded index
*/
@@ -256,19 +300,11 @@
if ($index === null) {
// Get mailbox data (UIDVALIDITY, counters, etc.) for status check
$mbox_data = $this->imap->folder_data($mailbox);
-
- if ($mbox_data['EXISTS']) {
- // get all threads (default sort order)
- $threads = $this->imap->fetch_threads($mailbox, true);
- }
- else {
- $threads = new rcube_result_thread($mailbox, '* THREAD');
- }
-
- $index['object'] = $threads;
+ // Get THREADS result
+ $index['object'] = $this->get_thread_data($mailbox, $mbox_data);
// insert/update
- $this->add_thread_row($mailbox, $threads, $mbox_data, $exists);
+ $this->add_thread_row($mailbox, $index['object'], $mbox_data, $exists);
}
$this->icache[$mailbox]['thread'] = $index;
@@ -291,27 +327,29 @@
return array();
}
- // Fetch messages from cache
- $sql_result = $this->db->query(
- "SELECT uid, data, flags"
- ." FROM ".$this->db->table_name('cache_messages')
- ." WHERE user_id = ?"
- ." AND mailbox = ?"
- ." AND uid IN (".$this->db->array2list($msgs, 'integer').")",
- $this->userid, $mailbox);
-
$msgs = array_flip($msgs);
$result = array();
- while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
- $uid = intval($sql_arr['uid']);
- $result[$uid] = $this->build_message($sql_arr);
+ if ($this->mode & self::MODE_MESSAGE) {
+ // Fetch messages from cache
+ $sql_result = $this->db->query(
+ "SELECT uid, data, flags"
+ ." FROM ".$this->db->table_name('cache_messages')
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?"
+ ." AND uid IN (".$this->db->array2list($msgs, 'integer').")",
+ $this->userid, $mailbox);
- if (!empty($result[$uid])) {
- // save memory, we don't need message body here (?)
- $result[$uid]->body = null;
+ while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+ $uid = intval($sql_arr['uid']);
+ $result[$uid] = $this->build_message($sql_arr);
- unset($msgs[$uid]);
+ if (!empty($result[$uid])) {
+ // save memory, we don't need message body here (?)
+ $result[$uid]->body = null;
+
+ unset($msgs[$uid]);
+ }
}
}
@@ -322,7 +360,10 @@
// Insert to DB and add to result list
if (!empty($messages)) {
foreach ($messages as $msg) {
- $this->add_message($mailbox, $msg, !array_key_exists($msg->uid, $result));
+ if ($this->mode & self::MODE_MESSAGE) {
+ $this->add_message($mailbox, $msg, !array_key_exists($msg->uid, $result));
+ }
+
$result[$msg->uid] = $msg;
}
}
@@ -353,23 +394,29 @@
return $this->icache['__message']['object'];
}
- $sql_result = $this->db->query(
- "SELECT flags, data"
- ." FROM ".$this->db->table_name('cache_messages')
- ." WHERE user_id = ?"
- ." AND mailbox = ?"
- ." AND uid = ?",
- $this->userid, $mailbox, (int)$uid);
+ if ($this->mode & self::MODE_MESSAGE) {
+ $sql_result = $this->db->query(
+ "SELECT flags, data"
+ ." FROM ".$this->db->table_name('cache_messages')
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?"
+ ." AND uid = ?",
+ $this->userid, $mailbox, (int)$uid);
- if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
- $message = $this->build_message($sql_arr);
- $found = true;
+ if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+ $message = $this->build_message($sql_arr);
+ $found = true;
+ }
}
// Get the message from IMAP server
if (empty($message) && $update) {
$message = $this->imap->get_message_headers($uid, $mailbox, true);
// cache will be updated in close(), see below
+ }
+
+ if (!($this->mode & self::MODE_MESSAGE)) {
+ return $message;
}
// Save the message in internal cache, will be written to DB in close()
@@ -407,8 +454,12 @@
return;
}
- $msg = serialize($this->db->encode(clone $message));
+ if (!($this->mode & self::MODE_MESSAGE)) {
+ return;
+ }
+
$flags = 0;
+ $msg = clone $message;
if (!empty($message->flags)) {
foreach ($this->flags as $idx => $flag) {
@@ -417,30 +468,49 @@
}
}
}
+
unset($msg->flags);
+ $msg = $this->db->encode($msg, true);
// update cache record (even if it exists, the update
// here will work as select, assume row exist if affected_rows=0)
if (!$force) {
$res = $this->db->query(
"UPDATE ".$this->db->table_name('cache_messages')
- ." SET flags = ?, data = ?, changed = ".$this->db->now()
+ ." SET flags = ?, data = ?, expires = " . ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
." WHERE user_id = ?"
." AND mailbox = ?"
." AND uid = ?",
$flags, $msg, $this->userid, $mailbox, (int) $message->uid);
- if ($this->db->affected_rows()) {
+ if ($this->db->affected_rows($res)) {
return;
}
}
+ $this->db->set_option('ignore_key_errors', true);
+
// insert new record
- $this->db->query(
+ $res = $this->db->query(
"INSERT INTO ".$this->db->table_name('cache_messages')
- ." (user_id, mailbox, uid, flags, changed, data)"
- ." VALUES (?, ?, ?, ?, ".$this->db->now().", ?)",
+ ." (user_id, mailbox, uid, flags, expires, data)"
+ ." VALUES (?, ?, ?, ?, ". ($this->ttl ? $this->db->now($this->ttl) : 'NULL') . ", ?)",
$this->userid, $mailbox, (int) $message->uid, $flags, $msg);
+
+ // race-condition, insert failed so try update (#1489146)
+ // thanks to ignore_key_errors "duplicate row" errors will be ignored
+ if ($force && !$res && !$this->db->is_error($res)) {
+ $this->db->query(
+ "UPDATE ".$this->db->table_name('cache_messages')
+ ." SET expires = " . ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
+ .", flags = ?, data = ?"
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?"
+ ." AND uid = ?",
+ $flags, $msg, $this->userid, $mailbox, (int) $message->uid);
+ }
+
+ $this->db->set_option('ignore_key_errors', false);
}
@@ -456,6 +526,10 @@
function change_flag($mailbox, $uids, $flag, $enabled = false)
{
if (empty($uids)) {
+ return;
+ }
+
+ if (!($this->mode & self::MODE_MESSAGE)) {
return;
}
@@ -481,11 +555,11 @@
$this->db->query(
"UPDATE ".$this->db->table_name('cache_messages')
- ." SET changed = ".$this->db->now()
+ ." SET expires = ". ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
.", flags = flags ".($enabled ? "+ $idx" : "- $idx")
." WHERE user_id = ?"
." AND mailbox = ?"
- .($uids !== null ? " AND uid IN (".$this->db->array2list($uids, 'integer').")" : "")
+ .(!empty($uids) ? " AND uid IN (".$this->db->array2list($uids, 'integer').")" : "")
." AND (flags & $idx) ".($enabled ? "= 0" : "= $idx"),
$this->userid, $mailbox);
}
@@ -499,6 +573,10 @@
*/
function remove_message($mailbox = null, $uids = null)
{
+ if (!($this->mode & self::MODE_MESSAGE)) {
+ return;
+ }
+
if (!strlen($mailbox)) {
$this->db->query(
"DELETE FROM ".$this->db->table_name('cache_messages')
@@ -604,23 +682,21 @@
/**
- * Delete cache entries older than TTL
- *
- * @param string $ttl Lifetime of message cache entries
+ * Delete expired cache entries
*/
- function expunge($ttl)
+ static function gc()
{
- // get expiration timestamp
- $ts = get_offset_time($ttl, -1);
+ $rcube = rcube::get_instance();
+ $db = $rcube->get_dbh();
- $this->db->query("DELETE FROM ".$this->db->table_name('cache_messages')
- ." WHERE changed < " . $this->db->fromunixtime($ts));
+ $db->query("DELETE FROM ".$db->table_name('cache_messages')
+ ." WHERE expires < " . $db->now());
- $this->db->query("DELETE FROM ".$this->db->table_name('cache_index')
- ." WHERE changed < " . $this->db->fromunixtime($ts));
+ $db->query("DELETE FROM ".$db->table_name('cache_index')
+ ." WHERE expires < " . $db->now());
- $this->db->query("DELETE FROM ".$this->db->table_name('cache_thread')
- ." WHERE changed < " . $this->db->fromunixtime($ts));
+ $db->query("DELETE FROM ".$db->table_name('cache_thread')
+ ." WHERE expires < " . $db->now());
}
@@ -639,7 +715,7 @@
if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
$data = explode('@', $sql_arr['data']);
- $index = @unserialize($data[0]);
+ $index = $this->db->decode($data[0], true);
unset($data[0]);
if (empty($index)) {
@@ -676,7 +752,7 @@
if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
$data = explode('@', $sql_arr['data']);
- $thread = @unserialize($data[0]);
+ $thread = $this->db->decode($data[0], true);
unset($data[0]);
if (empty($thread)) {
@@ -702,7 +778,7 @@
$data, $mbox_data = array(), $exists = false, $modseq = null)
{
$data = array(
- serialize($data),
+ $this->db->encode($data, true),
$sort_field,
(int) $this->skip_deleted,
(int) $mbox_data['UIDVALIDITY'],
@@ -712,20 +788,38 @@
$data = implode('@', $data);
if ($exists) {
- $sql_result = $this->db->query(
+ $res = $this->db->query(
"UPDATE ".$this->db->table_name('cache_index')
- ." SET data = ?, valid = 1, changed = ".$this->db->now()
+ ." SET data = ?, valid = 1, expires = " . ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?",
+ $data, $this->userid, $mailbox);
+
+ if ($this->db->affected_rows($res)) {
+ return;
+ }
+ }
+
+ $this->db->set_option('ignore_key_errors', true);
+
+ $res = $this->db->query(
+ "INSERT INTO ".$this->db->table_name('cache_index')
+ ." (user_id, mailbox, valid, expires, data)"
+ ." VALUES (?, ?, 1, ". ($this->ttl ? $this->db->now($this->ttl) : 'NULL') .", ?)",
+ $this->userid, $mailbox, $data);
+
+ // race-condition, insert failed so try update (#1489146)
+ // thanks to ignore_key_errors "duplicate row" errors will be ignored
+ if (!$exists && !$res && !$this->db->is_error($res)) {
+ $res = $this->db->query(
+ "UPDATE ".$this->db->table_name('cache_index')
+ ." SET data = ?, valid = 1, expires = " . ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
." WHERE user_id = ?"
." AND mailbox = ?",
$data, $this->userid, $mailbox);
}
- else {
- $sql_result = $this->db->query(
- "INSERT INTO ".$this->db->table_name('cache_index')
- ." (user_id, mailbox, data, valid, changed)"
- ." VALUES (?, ?, ?, 1, ".$this->db->now().")",
- $this->userid, $mailbox, $data);
- }
+
+ $this->db->set_option('ignore_key_errors', false);
}
@@ -735,28 +829,48 @@
private function add_thread_row($mailbox, $data, $mbox_data = array(), $exists = false)
{
$data = array(
- serialize($data),
+ $this->db->encode($data, true),
(int) $this->skip_deleted,
(int) $mbox_data['UIDVALIDITY'],
(int) $mbox_data['UIDNEXT'],
);
$data = implode('@', $data);
+ $expires = ($this->ttl ? $this->db->now($this->ttl) : 'NULL');
+
if ($exists) {
- $sql_result = $this->db->query(
+ $res = $this->db->query(
"UPDATE ".$this->db->table_name('cache_thread')
- ." SET data = ?, changed = ".$this->db->now()
+ ." SET data = ?, expires = $expires"
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?",
+ $data, $this->userid, $mailbox);
+
+ if ($this->db->affected_rows($res)) {
+ return;
+ }
+ }
+
+ $this->db->set_option('ignore_key_errors', true);
+
+ $res = $this->db->query(
+ "INSERT INTO ".$this->db->table_name('cache_thread')
+ ." (user_id, mailbox, expires, data)"
+ ." VALUES (?, ?, $expires, ?)",
+ $this->userid, $mailbox, $data);
+
+ // race-condition, insert failed so try update (#1489146)
+ // thanks to ignore_key_errors "duplicate row" errors will be ignored
+ if (!$exists && !$res && !$this->db->is_error($res)) {
+ $this->db->query(
+ "UPDATE ".$this->db->table_name('cache_thread')
+ ." SET expires = $expires, data = ?"
." WHERE user_id = ?"
." AND mailbox = ?",
$data, $this->userid, $mailbox);
}
- else {
- $sql_result = $this->db->query(
- "INSERT INTO ".$this->db->table_name('cache_thread')
- ." (user_id, mailbox, data, changed)"
- ." VALUES (?, ?, ?, ".$this->db->now().")",
- $this->userid, $mailbox, $data);
- }
+
+ $this->db->set_option('ignore_key_errors', false);
}
@@ -964,15 +1078,17 @@
$removed = array();
// Get known UIDs
- $sql_result = $this->db->query(
- "SELECT uid"
- ." FROM ".$this->db->table_name('cache_messages')
- ." WHERE user_id = ?"
- ." AND mailbox = ?",
- $this->userid, $mailbox);
+ if ($this->mode & self::MODE_MESSAGE) {
+ $sql_result = $this->db->query(
+ "SELECT uid"
+ ." FROM ".$this->db->table_name('cache_messages')
+ ." WHERE user_id = ?"
+ ." AND mailbox = ?",
+ $this->userid, $mailbox);
- while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
- $uids[] = $sql_arr['uid'];
+ while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+ $uids[] = $sql_arr['uid'];
+ }
}
// Synchronize messages data
@@ -983,7 +1099,7 @@
$uids, true, array('FLAGS'), $index['modseq'], $qresync);
if (!empty($result)) {
- foreach ($result as $id => $msg) {
+ foreach ($result as $msg) {
$uid = $msg->uid;
// Remove deleted message
if ($this->skip_deleted && !empty($msg->flags['DELETED'])) {
@@ -1004,7 +1120,7 @@
$this->db->query(
"UPDATE ".$this->db->table_name('cache_messages')
- ." SET flags = ?, changed = ".$this->db->now()
+ ." SET flags = ?, expires = " . ($this->ttl ? $this->db->now($this->ttl) : 'NULL')
." WHERE user_id = ?"
." AND mailbox = ?"
." AND uid = ?"
@@ -1032,17 +1148,18 @@
}
}
- // Invalidate thread index (?)
- if (!$index['valid']) {
- $this->remove_thread($mailbox);
- }
-
$sort_field = $index['sort_field'];
$sort_order = $index['object']->get_parameters('ORDER');
$exists = true;
// Validate index
if (!$this->validate($mailbox, $index, $exists)) {
+ // Invalidate (remove) thread index
+ // if $exists=false it was already removed in validate()
+ if ($exists) {
+ $this->remove_thread($mailbox);
+ }
+
// Update index
$data = $this->get_index_data($mailbox, $sort_field, $sort_order, $mbox_data);
}
@@ -1067,7 +1184,7 @@
*/
private function build_message($sql_arr)
{
- $message = $this->db->decode(unserialize($sql_arr['data']));
+ $message = $this->db->decode($sql_arr['data'], true);
if ($message) {
$message->flags = array();
@@ -1090,13 +1207,13 @@
// Save current message from internal cache
if ($message = $this->icache['__message']) {
// clean up some object's data
- $object = $this->message_object_prepare($message['object']);
+ $this->message_object_prepare($message['object']);
// calculate current md5 sum
- $md5sum = md5(serialize($object));
+ $md5sum = md5(serialize($message['object']));
if ($message['md5sum'] != $md5sum) {
- $this->add_message($message['mailbox'], $object, !$message['exists']);
+ $this->add_message($message['mailbox'], $message['object'], !$message['exists']);
}
$this->icache['__message']['md5sum'] = $md5sum;
@@ -1106,12 +1223,19 @@
/**
* Prepares message object to be stored in database.
+ *
+ * @param rcube_message_header|rcube_message_part
*/
- private function message_object_prepare($msg)
+ private function message_object_prepare(&$msg, &$size = 0)
{
- // Remove body too big (>25kB)
- if ($msg->body && strlen($msg->body) > 25 * 1024) {
- unset($msg->body);
+ // Remove body too big
+ if ($msg->body && ($length = strlen($msg->body))) {
+ $size += $length;
+
+ if ($size > $this->threshold * 1024) {
+ $size -= $length;
+ unset($msg->body);
+ }
}
// Fix mimetype which might be broken by some code when message is displayed
@@ -1121,13 +1245,19 @@
list($msg->ctype_primary, $msg->ctype_secondary) = explode('/', $msg->mimetype);
}
+ unset($msg->replaces);
+
if (is_array($msg->structure->parts)) {
- foreach ($msg->structure->parts as $idx => $part) {
- $msg->structure->parts[$idx] = $this->message_object_prepare($part);
+ foreach ($msg->structure->parts as $part) {
+ $this->message_object_prepare($part, $size);
}
}
- return $msg;
+ if (is_array($msg->parts)) {
+ foreach ($msg->parts as $part) {
+ $this->message_object_prepare($part, $size);
+ }
+ }
}
@@ -1150,6 +1280,25 @@
return $index;
}
+
+
+ /**
+ * Fetches thread data from IMAP server
+ */
+ private function get_thread_data($mailbox, $mbox_data = array())
+ {
+ if (empty($mbox_data)) {
+ $mbox_data = $this->imap->folder_data($mailbox);
+ }
+
+ if ($mbox_data['EXISTS']) {
+ // get all threads (default sort order)
+ return $this->imap->threads_direct($mailbox);
+ }
+
+ return new rcube_result_thread($mailbox, '* THREAD');
+ }
+
}
// for backward compat.
--
Gitblit v1.9.1