diff --git a/lib/Horde/ActiveSync/Collections.php b/lib/Horde/ActiveSync/Collections.php index 5b268285..13fac54f 100644 --- a/lib/Horde/ActiveSync/Collections.php +++ b/lib/Horde/ActiveSync/Collections.php @@ -1089,6 +1089,7 @@ public function save($preserve_folders = false) * * @throws Horde_ActiveSync_Exception_InvalidRequest * @throws Horde_ActiveSync_Exception_FolderGone + * @throws Horde_ActiveSync_Exception_StaleState */ public function initCollectionState(array &$collection, $requireSyncKey = false) { @@ -1239,6 +1240,23 @@ public function pollForChanges($heartbeat, $interval, array $options = []) // Initialize the collection's state data in the state handler. try { $this->initCollectionState($collection, true); + } catch (Horde_ActiveSync_Exception_StaleState $e) { + $this->_logger->notice( + sprintf( + 'COLLECTIONS: Corrupt state for %s; forcing collection resync: %s', + $id, + $e->getMessage() + ) + ); + $this->_as->state->loadState( + [], + null, + Horde_ActiveSync::REQUEST_TYPE_SYNC, + $id + ); + $this->setGetChangesFlag($id); + $dataavailable = true; + continue; } catch (Horde_ActiveSync_Exception_StateGone $e) { if (!empty($options['pingable'])) { $this->_logger->notice( diff --git a/lib/Horde/ActiveSync/Connector/Exporter/Sync.php b/lib/Horde/ActiveSync/Connector/Exporter/Sync.php index 6dbab318..21df082f 100644 --- a/lib/Horde/ActiveSync/Connector/Exporter/Sync.php +++ b/lib/Horde/ActiveSync/Connector/Exporter/Sync.php @@ -52,10 +52,13 @@ public function setChanges($changes, $collection = null) /** * Sends the next change in the set to the client. + * Send the next change to the client. * - * @return boolean|Horde_Exception True if more changes can be sent false if - * all changes were sent, Horde_Exception if - * there was an error sending an item. + * @return boolean True if a change was exported successfully. False if + * there are no more changes or the export must stop + * (caller should break out of the send loop). + * Throws on temporary/fatal failures instead of returning + * an exception object. */ public function sendNextChange() { @@ -452,7 +455,9 @@ protected function _getNextChange() /** * Sends the next message change to the client. * - * @return @see self::sendNextChange() + * @return boolean True on successful export, false to stop the loop. + * + * @throws Horde_ActiveSync_Exception_TemporaryFailure */ protected function _sendNextChange() { @@ -493,9 +498,12 @@ protected function _sendNextChange() 'Message gone or error reading message from server: %s', $e->getMessage() )); + // Permanently unavailable on the mail server (expunged, + // corrupt, or otherwise unreadable). Drop from pending + // and continue exporting the rest of the batch. $this->_as->state->updateState($change['type'], $change); $this->_step++; - return $e; + return true; } catch (Horde_ActiveSync_Exception_TemporaryFailure $e) { $this->_logger->err( sprintf( @@ -511,9 +519,11 @@ protected function _sendNextChange() $e->getMessage() ) ); - $this->_as->state->updateState($change['type'], $change); - $this->_step++; - return $e; + /* Do not drop pending changes on export failure. + * Return false to break the caller's send loop; the + * remaining batch stays in sync_pending for the next + * SYNC request. */ + return false; } break; diff --git a/lib/Horde/ActiveSync/Folder/Imap.php b/lib/Horde/ActiveSync/Folder/Imap.php index a485c274..047dac6d 100644 --- a/lib/Horde/ActiveSync/Folder/Imap.php +++ b/lib/Horde/ActiveSync/Folder/Imap.php @@ -102,6 +102,20 @@ class Horde_ActiveSync_Folder_Imap extends Horde_ActiveSync_Folder_Base implemen */ protected $_primed = false; + /** + * PING watermark (IMAP STATUS last acknowledged to the client). + * + * Part of the three-state email sync model: + * - $_status / modseq() — SYNC watermark (CHANGEDSINCE) + * - $_pingStatus — PING watermark (this property) + * - horde_activesync_state.sync_pending — MOREAVAILABLE batch (SYNC only) + * + * @see Horde_ActiveSync_State_Base::getChanges() for the PING vs SYNC paths. + * + * @var array + */ + protected $_pingStatus = []; + /** * Set message changes. * @@ -269,6 +283,7 @@ public function setRemoved(array $uids) */ public function updateState() { + $deferInitialComplete = false; if (empty($this->_status[self::HIGHESTMODSEQ])) { $this->_messages = array_diff(array_keys($this->_messages), $this->_removed); foreach ($this->_added as $add) { @@ -277,8 +292,11 @@ public function updateState() $this->_messages = $this->_flags + array_flip($this->_messages); } else { if ($this->_primed) { - $this->_messages = $this->_added; + // CONDSTORE initial priming: the full UID list is exported via + // sync_pending. Only add UIDs to _messages as they are actually + // sent to the client (acknowledgeExportedMessage()). $this->_primed = false; + $deferInitialComplete = true; } else { foreach ($this->_added as $add) { $this->_messages[] = $add; @@ -293,9 +311,138 @@ public function updateState() $this->_changed = []; $this->_flags = []; $this->_softDeleted = []; + if (!$deferInitialComplete) { + $this->haveInitialSync = true; + } + $this->_syncPingStatus(); + } + + /** + * Track a message UID successfully exported to the client. + * + * During CONDSTORE initial sync the server's _messages cache must reflect + * only mail the client has actually received, not the full primed UID + * list polled from IMAP. + * + * @param integer|string $uid The IMAP UID. + */ + public function acknowledgeExportedMessage($uid) + { + if ($uid === '' || $uid === null) { + return; + } + + if (empty($this->_status[self::HIGHESTMODSEQ])) { + $this->_messages[$uid] = $uid; + } else { + $this->_messages[] = $uid; + } + } + + /** + * Mark an initial folder sync as complete. + * + * Called once sync_pending is empty and all MOREAVAILABLE batches have + * been delivered to the client. + */ + public function markInitialSyncComplete() + { $this->haveInitialSync = true; } + /** + * Return the MODSEQ value used for PING change detection. + * + * @return integer + */ + public function pingModseq() + { + if (!empty($this->_pingStatus[self::HIGHESTMODSEQ])) { + return $this->_pingStatus[self::HIGHESTMODSEQ]; + } + + return $this->modseq(); + } + + /** + * Return the UIDNEXT value used for PING change detection. + * + * @return integer + */ + public function pingUidnext() + { + if (array_key_exists(self::UIDNEXT, $this->_pingStatus)) { + return $this->_pingStatus[self::UIDNEXT]; + } + + return $this->uidnext(); + } + + /** + * Return the message count used for PING change detection. + * + * @return integer + */ + public function pingTotalMessages() + { + if (array_key_exists(self::MESSAGES, $this->_pingStatus)) { + return $this->_pingStatus[self::MESSAGES]; + } + + return $this->total_messages(); + } + + /** + * Advance the PING checkpoint to the current IMAP STATUS. + * + * @param array $status An IMAP STATUS result. + */ + public function acknowledgePingStatus(array $status) + { + $this->_pingStatus = [ + self::UIDVALIDITY => $status[self::UIDVALIDITY] ?? $this->uidvalidity(), + self::UIDNEXT => $status[self::UIDNEXT] ?? $this->uidnext(), + self::HIGHESTMODSEQ => $status[self::HIGHESTMODSEQ] ?? 0, + self::MESSAGES => $status[self::MESSAGES] ?? 0, + ]; + } + + /** + * Align the PING checkpoint with the current SYNC state. + */ + protected function _syncPingStatus() + { + $synced = [ + self::UIDVALIDITY => $this->uidvalidity(), + self::UIDNEXT => $this->uidnext(), + self::HIGHESTMODSEQ => $this->modseq(), + self::MESSAGES => $this->total_messages(), + ]; + + // Never move the PING checkpoint backward. Partial SYNC responses can + // leave the SYNC modseq behind the server while the PING checkpoint was + // already advanced during PING. + if (empty($this->_pingStatus)) { + $this->_pingStatus = $synced; + return; + } + + if (!empty($synced[self::HIGHESTMODSEQ]) + && (empty($this->_pingStatus[self::HIGHESTMODSEQ]) + || $synced[self::HIGHESTMODSEQ] > $this->_pingStatus[self::HIGHESTMODSEQ])) { + $this->_pingStatus[self::HIGHESTMODSEQ] = $synced[self::HIGHESTMODSEQ]; + } + if ($synced[self::UIDNEXT] > ($this->_pingStatus[self::UIDNEXT] ?? 0)) { + $this->_pingStatus[self::UIDNEXT] = $synced[self::UIDNEXT]; + } + if ($synced[self::MESSAGES] != ($this->_pingStatus[self::MESSAGES] ?? null)) { + $this->_pingStatus[self::MESSAGES] = $synced[self::MESSAGES]; + } + if (!empty($synced[self::UIDVALIDITY])) { + $this->_pingStatus[self::UIDVALIDITY] = $synced[self::UIDVALIDITY]; + } + } + /** * Return the folder's UID validity. * @@ -441,17 +588,21 @@ public function serialize() $msgs = $this->_messages; } - return json_encode( - [ - 's' => $this->_status, - 'm' => $msgs, - 'f' => $this->_serverid, - 'c' => $this->_class, - 'lsd' => $this->_lastSinceDate, - 'sd' => $this->_softDelete, - 'hi' => $this->haveInitialSync, - 'v' => self::VERSION] - ); + $data = [ + 's' => $this->_status, + 'm' => $msgs, + 'f' => $this->_serverid, + 'c' => $this->_class, + 'lsd' => $this->_lastSinceDate, + 'sd' => $this->_softDelete, + 'hi' => $this->haveInitialSync, + 'v' => self::VERSION, + ]; + if (!empty($this->_pingStatus)) { + $data['ps'] = $this->_pingStatus; + } + + return json_encode($data); } /** @@ -477,7 +628,10 @@ public function unserialize($data) $this->_class = $d_data['c']; $this->_lastSinceDate = $d_data['lsd']; $this->_softDelete = $d_data['sd']; - $this->haveInitialSync = empty($d_data['hi']) ? !empty($this->_messages) : $d_data['hi']; + $this->haveInitialSync = array_key_exists('hi', $d_data) + ? (bool) $d_data['hi'] + : !empty($this->_messages); + $this->_pingStatus = !empty($d_data['ps']) ? $d_data['ps'] : []; if (!empty($this->_status[self::HIGHESTMODSEQ]) && is_string($this->_messages)) { $this->_messages = $this->_fromSequenceString($this->_messages); diff --git a/lib/Horde/ActiveSync/Imap/Adapter.php b/lib/Horde/ActiveSync/Imap/Adapter.php index 57589bcd..62768835 100644 --- a/lib/Horde/ActiveSync/Imap/Adapter.php +++ b/lib/Horde/ActiveSync/Imap/Adapter.php @@ -437,6 +437,12 @@ public function getMessages($folderid, array $messages, array $options = []) try { $ret[] = $this->_buildMailMessage($mbox, $data, $options); } catch (Horde_Exception_NotFound $e) { + $this->_logger->notice(sprintf( + 'Unable to build message UID %s in %s: %s', + $data->getUid(), + $folderid, + $e->getMessage() + )); } } } @@ -568,19 +574,24 @@ public function ping(Horde_ActiveSync_Folder_Imap $folder) ) ); - // If we have per mailbox MODSEQ then we can pick up flag changes too. + // PING path: compare against $_pingStatus, not SYNC modseq() in + // $_status. See Horde_ActiveSync_State_Base::getChanges() and + // Horde_ActiveSync_Folder_Imap::$_pingStatus. $modseq = $status[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ]; - if ($modseq && $folder->modseq() > 0 && $folder->modseq() < $modseq) { + if ($modseq && $folder->pingModseq() > 0 && $folder->pingModseq() < $modseq) { + $folder->acknowledgePingStatus($status); return true; } // Increase in UIDNEXT is always a positive PING. - if ($folder->uidnext() < $status['uidnext']) { + if ($folder->pingUidnext() < $status['uidnext']) { + $folder->acknowledgePingStatus($status); return true; } // If the message count changes, something certainly changed. - if ($folder->total_messages() != $status['messages']) { + if ($folder->pingTotalMessages() != $status['messages']) { + $folder->acknowledgePingStatus($status); return true; } diff --git a/lib/Horde/ActiveSync/Imap/MessageBodyData.php b/lib/Horde/ActiveSync/Imap/MessageBodyData.php index edd90177..7bcdb0e0 100644 --- a/lib/Horde/ActiveSync/Imap/MessageBodyData.php +++ b/lib/Horde/ActiveSync/Imap/MessageBodyData.php @@ -376,6 +376,65 @@ protected function _wantPlainText($html_id, $want_html) * Horde_ActiveSync_Exception_EmailFatalFailure */ protected function _fetchData(array $params) + { + $data = $this->_tryFetchBodyData($params, true); + if (!$data) { + // Some IMAP servers (notably Dovecot) return no data when + // BODY[x.y.z] and BODY[x.y.z].SIZE are requested together for + // deeply nested MIME parts. Fall back to fetching content only. + $data = $this->_tryFetchBodyData($params, false); + } + if (!$data) { + throw new Horde_Exception_NotFound( + sprintf('Could not load message %s from server.', $this->_uid) + ); + } + + return $data; + } + + /** + * Attempt to fetch body part data from the IMAP server. + * + * @param array $params Parameter array. + * - html_id (string) The MIME id of the HTML part, if any. + * - text_id (string) The MIME id of the plain part, if any. + * @param boolean $withSize If true, also request RFC822.SIZE for parts. + * + * @return Horde_Imap_Client_Data_Fetch|null The results, or null. + * @throws Horde_ActiveSync_Exception, + * Horde_ActiveSync_Exception_TemporaryFailure + */ + protected function _tryFetchBodyData(array $params, $withSize) + { + try { + $fetch_ret = $this->_imap->fetch( + $this->_mbox, + $this->_buildBodyFetchQuery($params, $withSize), + ['ids' => new Horde_Imap_Client_Ids([$this->_uid])] + ); + } catch (Horde_Imap_Client_Exception $e) { + // If we lost the connection, don't continue to try. + if ($e->getCode() == Horde_Imap_Client_Exception::DISCONNECT) { + throw new Horde_ActiveSync_Exception_TemporaryFailure($e->getMessage()); + } + throw new Horde_ActiveSync_Exception($e); + } + + return $fetch_ret->first() ?: null; + } + + /** + * Build a FETCH query for message body part data. + * + * @param array $params Parameter array. + * - html_id (string) The MIME id of the HTML part, if any. + * - text_id (string) The MIME id of the plain part, if any. + * @param boolean $withSize If true, also request RFC822.SIZE for parts. + * + * @return Horde_Imap_Client_Fetch_Query + */ + protected function _buildBodyFetchQuery(array $params, $withSize) { $query = new Horde_Imap_Client_Fetch_Query(); $query_opts = [ @@ -386,38 +445,26 @@ protected function _fetchData(array $params) // Get body information if ($this->_version >= Horde_ActiveSync::VERSION_TWELVE) { if (!empty($params['html_id'])) { - $query->bodyPartSize($params['html_id']); + if ($withSize) { + $query->bodyPartSize($params['html_id']); + } $query->bodyPart($params['html_id'], $query_opts); } if (!empty($params['text_id'])) { $query->bodyPart($params['text_id'], $query_opts); - $query->bodyPartSize($params['text_id']); + if ($withSize) { + $query->bodyPartSize($params['text_id']); + } } } else { // EAS 2.5 Plaintext body $query->bodyPart($params['text_id'], $query_opts); - $query->bodyPartSize($params['text_id']); - } - try { - $fetch_ret = $this->_imap->fetch( - $this->_mbox, - $query, - ['ids' => new Horde_Imap_Client_Ids([$this->_uid])] - ); - } catch (Horde_Imap_Client_Exception $e) { - // If we lost the connection, don't continue to try. - if ($e->getCode() == Horde_Imap_Client_Exception::DISCONNECT) { - throw new Horde_ActiveSync_Exception_TemporaryFailure($e->getMessage()); + if ($withSize) { + $query->bodyPartSize($params['text_id']); } - throw new Horde_ActiveSync_Exception($e); - } - if (!$data = $fetch_ret->first()) { - throw new Horde_Exception_NotFound( - sprintf('Could not load message %s from server.', $this->_uid) - ); } - return $data; + return $query; } /** diff --git a/lib/Horde/ActiveSync/Mime.php b/lib/Horde/ActiveSync/Mime.php index def0390a..1cd12bb8 100644 --- a/lib/Horde/ActiveSync/Mime.php +++ b/lib/Horde/ActiveSync/Mime.php @@ -132,6 +132,7 @@ public function isAttachment($id, $mime_type) return true; } return false; + case 'application/pgp-keys': case 'application/pkcs7-signature': case 'application/x-pkcs7-signature': return false; diff --git a/lib/Horde/ActiveSync/Mime/Iterator.php b/lib/Horde/ActiveSync/Mime/Iterator.php index f54fd026..430b49ef 100644 --- a/lib/Horde/ActiveSync/Mime/Iterator.php +++ b/lib/Horde/ActiveSync/Mime/Iterator.php @@ -88,6 +88,7 @@ protected function _isAttachment($part) return true; } return false; + case 'application/pgp-keys': case 'application/pkcs7-signature': case 'application/x-pkcs7-signature': return false; diff --git a/lib/Horde/ActiveSync/Request/Sync.php b/lib/Horde/ActiveSync/Request/Sync.php index 7013ac70..e6bf4e81 100644 --- a/lib/Horde/ActiveSync/Request/Sync.php +++ b/lib/Horde/ActiveSync/Request/Sync.php @@ -334,6 +334,19 @@ protected function _handle() // Initialize this collection's state. try { $this->_collections->initCollectionState($collection); + } catch (Horde_ActiveSync_Exception_StaleState $e) { + $this->_logger->err(sprintf( + 'Force resetting state for %s: %s', + $id, + $e->getMessage() + )); + $this->_state->loadState( + [], + null, + Horde_ActiveSync::REQUEST_TYPE_SYNC, + $id + ); + $statusCode = self::STATUS_KEYMISM; } catch (Horde_ActiveSync_Exception_StateGone $e) { $this->_logger->warn('SYNC terminating, state not found'); $statusCode = self::STATUS_KEYMISM; @@ -470,19 +483,23 @@ protected function _handle() $exporter->setChanges($this->_collections->getCollectionChanges(false), $collection); $this->_encoder->startTag(Horde_ActiveSync::SYNC_COMMANDS); $cnt_collection = 0; + /* sendNextChange() returns true on successful export, + * false when no more changes remain or on non-fatal + * error (remaining batch preserved in sync_pending). */ while ($cnt_collection < $max_windowsize - && $cnt_global < $this->_collections->getDefaultWindowSize() - && $progress = $exporter->sendNextChange()) { + && $cnt_global < $this->_collections->getDefaultWindowSize()) { + $progress = $exporter->sendNextChange(); + if ($progress !== true) { + break; + } $this->_logger->meta( sprintf( 'Peak memory usage after message: %d', memory_get_peak_usage(true) ) ); - if ($progress === true) { - ++$cnt_collection; - ++$cnt_global; - } + ++$cnt_collection; + ++$cnt_global; } $this->_encoder->endTag(); } @@ -836,8 +853,14 @@ protected function _parseSyncCommands(&$collection) return false; } catch (Horde_ActiveSync_Exception_StaleState $e) { $this->_logger->notice($e->getMessage()); - $this->_statusCode = self::STATUS_SERVERERROR; - $this->_handleGlobalSyncError(); + $this->_state->loadState( + [], + null, + Horde_ActiveSync::REQUEST_TYPE_SYNC, + $collection['id'] + ); + $this->_statusCode = self::STATUS_KEYMISM; + $this->_handleError($collection); return false; } catch (Horde_ActiveSync_Exception_FolderGone $e) { $this->_logger->notice($e->getMessage()); diff --git a/lib/Horde/ActiveSync/State/Base.php b/lib/Horde/ActiveSync/State/Base.php index 8893e2ce..99e5cecf 100644 --- a/lib/Horde/ActiveSync/State/Base.php +++ b/lib/Horde/ActiveSync/State/Base.php @@ -20,6 +20,11 @@ */ abstract class Horde_ActiveSync_State_Base { + /** + * Treat committed collection lock tokens older than this as stale (seconds). + */ + public const STATE_ROW_LOCK_STALE_SECONDS = 300; + /** * Configuration parameters * @@ -107,6 +112,17 @@ abstract class Horde_ActiveSync_State_Base */ protected $_changes; + /** + * Serialized sync_pending payload last read from storage. + * + * Used when persisting a PING checkpoint: PING may update sync_data + * (folder + PING watermark) but must not clear an in-flight MOREAVAILABLE + * batch in sync_pending. + * + * @var string|Horde_Db_Value_Binary|null + */ + protected $_syncPendingBlob; + /** * The type of request we are handling. * @@ -137,6 +153,141 @@ public function __construct(array $params = []) $this->_procid = getmypid(); } + /** + * Validate deserialized sync_data for a collection SYNC request. + * + * Missing or unparseable blobs (unserialize() === false) are treated as + * absent state and rebuilt on load. Clearly corrupt payloads (arrays, + * FOLDERSYNC-shaped blobs, sync_pending-shaped data) raise StaleState so + * callers can force a collection resync via STATUS_KEYMISMATCH. + * + * @param mixed $data Result of unserialize() on sync_data. + * @param string|null $rawBlob Raw sync_data string for logging. + * + * @return Horde_ActiveSync_Folder_Base|false Folder object, or false when + * state is simply missing. + * + * @throws Horde_ActiveSync_Exception_StaleState + */ + protected function _normalizeSyncFolderData($data, $rawBlob = null) + { + if ($this->_type != Horde_ActiveSync::REQUEST_TYPE_SYNC) { + return $data; + } + + if ($data === false) { + return false; + } + + if ($data instanceof Horde_ActiveSync_Folder_Base) { + return $data; + } + + $collectionId = !empty($this->_collection['id']) + ? $this->_collection['id'] + : 'unknown'; + $head = ($rawBlob !== null && $rawBlob !== '') + ? substr($rawBlob, 0, 60) + : (is_object($data) ? get_class($data) : gettype($data)); + + $this->_logger->warn( + sprintf( + 'STATE: Invalid sync_data for collection %s (synckey %s); forcing collection resync.', + $collectionId, + $this->_syncKey + ) + ); + $this->_logger->meta( + sprintf( + 'STATE: Invalid sync_data details: type=%s, head=%s', + is_object($data) ? get_class($data) : gettype($data), + $head + ) + ); + + throw new Horde_ActiveSync_Exception_StaleState( + sprintf( + 'Corrupt sync_data for collection %s (synckey %s, head=%s).', + $collectionId, + $this->_syncKey, + $head + ) + ); + } + + /** + * Refuse to persist FOLDERSYNC-shaped or empty collection sync_data blobs. + * + * @param string $data Serialized sync_data about to be written. + * + * @throws Horde_ActiveSync_Exception_StaleState + */ + protected function _assertSyncDataBlob($data) + { + if ($this->_type != Horde_ActiveSync::REQUEST_TYPE_SYNC) { + return; + } + + if ($data === '' || $data === 'a:0:{}' || preg_match('/^a:\d+:\{/', $data)) { + throw new Horde_ActiveSync_Exception_StaleState( + sprintf( + 'Refusing to persist corrupt sync_data blob for collection %s (synckey %s, head=%s).', + !empty($this->_collection['id']) ? $this->_collection['id'] : 'unknown', + $this->_syncKey, + substr($data, 0, 40) + ) + ); + } + } + + /** + * Create an empty folder object for the current collection. + * + * @return Horde_ActiveSync_Folder_Base + */ + protected function _createEmptySyncFolder() + { + if (!empty($this->_collection['class']) + && $this->_collection['class'] == Horde_ActiveSync::CLASS_EMAIL) { + return new Horde_ActiveSync_Folder_Imap( + $this->_collection['serverid'], + Horde_ActiveSync::CLASS_EMAIL + ); + } + + if (!empty($this->_collection['serverid']) + && $this->_collection['serverid'] == 'RI') { + return new Horde_ActiveSync_Folder_RI('RI', 'RI'); + } + + return new Horde_ActiveSync_Folder_Collection( + $this->_collection['serverid'], + $this->_collection['class'] + ); + } + + /** + * Refuse to persist corrupt collection sync_data. + * + * @throws Horde_ActiveSync_Exception_StaleState + */ + protected function _assertValidSyncFolderBeforeSave() + { + if ($this->_type != Horde_ActiveSync::REQUEST_TYPE_SYNC) { + return; + } + + if (!isset($this->_folder) || !($this->_folder instanceof Horde_ActiveSync_Folder_Base)) { + throw new Horde_ActiveSync_Exception_StaleState( + sprintf( + 'Refusing to save invalid sync_data for collection %s (synckey %s).', + !empty($this->_collection['id']) ? $this->_collection['id'] : 'unknown', + $this->_syncKey + ) + ); + } + } + /** * Update the $oldKey syncState to $newKey. * @@ -363,7 +514,27 @@ public function getFolderUidForBackendId($serverid) } /** - * Get all items that have changed since the last sync time + * Get all items that have changed since the last sync time. + * + * Email collections use two deliberately separate paths (do not merge): + * + * SYNC ($options['ping'] === false): + * - Uses Horde_ActiveSync_Folder_Imap::$_status / modseq() for + * CHANGEDSINCE against the IMAP server. + * - May resume from sync_pending (a MOREAVAILABLE batch not yet sent). + * - Calls updateState() after changes are exported. + * + * PING ($options['ping'] === true): + * - MUST NOT treat sync_pending as a change signal (it is an in-flight + * SYNC batch; reusing it causes infinite PING loops — see BigFamily). + * - Uses $_pingStatus via Horde_ActiveSync_Imap_Adapter::ping() for a + * lightweight IMAP STATUS comparison. + * - Advances $_pingStatus on detection and persists via save() without + * touching sync_pending or calling updateState() (SYNC modseq must + * stay behind until the client completes SYNC). + * + * @see Horde_ActiveSync_Folder_Imap::$_pingStatus + * @see Horde_ActiveSync_State_Sql — sync_pending column * * @param array $options An options array: * - ping: (boolean) Only detect if there is a change, do not build @@ -403,8 +574,11 @@ public function getChanges(array $options = []) ) ); - // Check for previously found changes first. - if (!empty($this->_changes)) { + // SYNC may resume a MOREAVAILABLE batch from sync_pending. PING must + // always poll IMAP STATUS instead — pending is not a PING signal. + if (!empty($options['ping'])) { + $this->_changes = null; + } elseif (!empty($this->_changes)) { $this->_logger->meta('STATE: Returning previously found changes.'); return $this->_changes; } @@ -457,6 +631,13 @@ public function getChanges(array $options = []) // Only update the folderstate if we are not PINGing. if (empty($options['ping'])) { $this->_folder->updateState(); + } elseif ($this->_folder instanceof Horde_ActiveSync_Folder_Imap + && !empty($this->_collection['class']) + && $this->_collection['class'] == Horde_ActiveSync::CLASS_EMAIL + && count($changes)) { + // Persist sync_data (folder + PING watermark) only. sync_pending + // must survive for the client's in-flight MOREAVAILABLE batch. + $this->save(['preservePending' => true]); } $this->_logger->meta( @@ -869,13 +1050,16 @@ public static function RowCmp($a, $b) * @param string $id The folder id this state represents. If empty * assumed to be a foldersync state. * - * @throws Horde_ActiveSync_Exception, Horde_ActiveSync_Exception_StateGone + * @throws Horde_ActiveSync_Exception + * @throws Horde_ActiveSync_Exception_StateGone + * @throws Horde_ActiveSync_Exception_StaleState */ public function loadState(array $collection, $syncKey, $type = null, $id = null) { // Initialize the local members. $this->_collection = $collection; $this->_changes = null; + $this->_syncPendingBlob = null; $this->_type = $type; // If this is a FOLDERSYNC, mock the device id. @@ -900,7 +1084,17 @@ public function loadState(array $collection, $syncKey, $type = null, $id = null) : ($this->_collection['serverid'] == 'RI' ? new Horde_ActiveSync_Folder_RI('RI', 'RI') : new Horde_ActiveSync_Folder_Collection($this->_collection['serverid'], $this->_collection['class'])); } $this->_syncKey = '0'; - $this->_resetDeviceState($id); + $lockFolderId = ($type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) + ? Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC + : $id; + $this->_acquireCollectionLock($lockFolderId); + try { + $this->_resetDeviceState($id); + $this->_releaseCollectionLock(true); + } catch (Throwable $e) { + $this->_releaseCollectionLock(false); + throw $e; + } return; } @@ -914,11 +1108,17 @@ public function loadState(array $collection, $syncKey, $type = null, $id = null) } $this->_syncKey = $syncKey; - // Cleanup older syncstates - $this->_gc($syncKey); + $this->_acquireCollectionLock(); + try { + // Cleanup older syncstates + $this->_gc($syncKey); - // Load the state - $this->_loadState(); + // Load the state + $this->_loadState(); + } catch (Throwable $e) { + $this->_releaseCollectionLock(false); + throw $e; + } } /** @@ -949,6 +1149,28 @@ protected function _loadState() throw new Horde_ActiveSync_Exception('Must be implemented in concrete class.'); } + /** + * Acquire an exclusive lock for the current collection. + * + * Serializes SYNC/PING workers across sync_key rows for the same folder. + * No-op in drivers that do not implement collection locking. + * + * @param string|null $folderId Optional folder id override. + */ + protected function _acquireCollectionLock($folderId = null) + { + } + + /** + * Release a collection lock acquired by _acquireCollectionLock(). + * + * @param boolean $commit Commit (true) or roll back (false) the lock + * transaction when this instance owns it. + */ + protected function _releaseCollectionLock($commit = false) + { + } + /** * Check for the existence of ANY entries in the map table for this device * and user. @@ -992,9 +1214,51 @@ public function updateSyncStamp() } /** - * Save the current syncstate to storage + * Save the current syncstate to storage. + * + * @param array $options Options array: + * - preservePending: (boolean) Write sync_data but keep the sync_pending + * column as loaded from storage. Used when persisting + * a PING checkpoint. DEFAULT: false. + */ + abstract public function save(array $options = []); + + /** + * Track a successfully exported message in the folder cache. + * + * During CONDSTORE initial sync the folder's _messages list must reflect + * only mail the client has actually received. + * + * @param string $type A Horde_ActiveSync::CHANGE_TYPE_* constant. + * @param array $change The change hash being exported. + */ + protected function _acknowledgeExportedChange($type, array $change) + { + if (!$this->_folder instanceof Horde_ActiveSync_Folder_Imap) { + return; + } + + switch ($type) { + case Horde_ActiveSync::CHANGE_TYPE_CHANGE: + case Horde_ActiveSync::CHANGE_TYPE_DRAFT: + if (!empty($change['id'])) { + $this->_folder->acknowledgeExportedMessage($change['id']); + } + break; + } + } + + /** + * Mark initial folder sync complete once sync_pending is drained. */ - abstract public function save(); + protected function _finalizeInitialSyncIfComplete() + { + if (empty($this->_changes) + && $this->_folder instanceof Horde_ActiveSync_Folder_Imap + && !$this->_folder->haveInitialSync) { + $this->_folder->markInitialSyncComplete(); + } + } /** * Update the state to reflect changes diff --git a/lib/Horde/ActiveSync/State/Mongo.php b/lib/Horde/ActiveSync/State/Mongo.php index 6f023934..1f0e5f27 100644 --- a/lib/Horde/ActiveSync/State/Mongo.php +++ b/lib/Horde/ActiveSync/State/Mongo.php @@ -76,6 +76,7 @@ class Horde_ActiveSync_State_Mongo extends Horde_ActiveSync_State_Base implement public const COLLECTION_MAP = 'HAS_map'; public const COLLECTION_DEVICE = 'HAS_device'; public const COLLECTION_STATE = 'HAS_state'; + public const COLLECTION_LOCK = 'HAS_collection_lock'; /** Field names **/ public const MONGO_ID = '_id'; @@ -97,6 +98,9 @@ class Horde_ActiveSync_State_Mongo extends Horde_ActiveSync_State_Base implement public const SYNC_MOD = 'sync_mod'; public const SYNC_PENDING = 'sync_pending'; public const SYNC_TIMESTAMP = 'sync_timestamp'; + public const SYNC_LOCK = 'sync_lock'; + public const LOCK_TOKEN = 'lock_token'; + public const LOCK_TIME = 'lock_time'; public const DEVICE_ID = 'device_id'; public const DEVICE_TYPE = 'device_type'; public const DEVICE_AGENT = 'device_agent'; @@ -186,6 +190,52 @@ class Horde_ActiveSync_State_Mongo extends Horde_ActiveSync_State_Base implement 'id' => self::DEVICE_ID, ]; + /** + * Minimum sync_mod delta before updateSyncStamp() persists a new value. + */ + public const SYNCSTAMP_UPDATE_THRESHOLD = 30000; + + /** + * Treat sync_lock values older than this as stale (seconds). + */ + public const STATE_ROW_LOCK_STALE_SECONDS = 300; + + /** + * True while a document lock is held for the loaded sync_key (released on + * save() or updateSyncStamp()). + * + * @var boolean + */ + protected $_stateRowLockHeld = false; + + /** + * sync_lock value set when this instance acquired the document lock. + * + * @var integer|null + */ + protected $_stateLockToken = null; + + /** + * True while a collection lock document is held. + * + * @var boolean + */ + protected $_collectionLockHeld = false; + + /** + * lock_token value set when this instance acquired the collection lock. + * + * @var integer|null + */ + protected $_collectionLockToken = null; + + /** + * Folder id used for the active collection lock, if any. + * + * @var string|null + */ + protected $_collectionLockFolderId = null; + /** * Const'r * @@ -205,6 +255,346 @@ public function __construct(array $params = []) $this->_db = $this->_mongo->selectDb(null); } + /** + * Release any document lock left open when the request ends without save(). + */ + public function __destruct() + { + if ($this->_stateRowLockHeld) { + $this->_releaseStateRowLock(false); + } + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(false); + } + } + + /** + * HAS_collection_lock collection for per-collection serialization. + * + * @return MongoCollection + */ + protected function _collectionLockCollection() + { + return $this->_db->selectCollection(self::COLLECTION_LOCK); + } + + /** + * HAS_state collection for the current sync key. + * + * @return MongoCollection + */ + protected function _stateCollection() + { + return $this->_db->selectCollection(self::COLLECTION_STATE); + } + + /** + * Base query for the sync state document being loaded or saved. + * + * @return array + */ + protected function _stateRowLockQuery() + { + return [ + self::MONGO_ID => $this->_syncKey, + self::SYNC_FOLDERID => $this->_collection['id'], + ]; + } + + /** + * Release a document lock opened by _loadState(). + * + * @param boolean $commit Unused for Mongo; kept for parity with Sql. + */ + protected function _releaseStateRowLock($commit = false) + { + if (!$this->_stateRowLockHeld) { + return; + } + + try { + $this->_stateCollection()->update( + array_merge( + $this->_stateRowLockQuery(), + [self::SYNC_LOCK => $this->_stateLockToken] + ), + ['$unset' => [self::SYNC_LOCK => '']] + ); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + } + + $this->_stateRowLockHeld = false; + $this->_stateLockToken = null; + } + + /** + * Atomically lock the current sync state document and return its fields. + * + * @return array + * + * @throws Horde_ActiveSync_Exception + * @throws Horde_ActiveSync_Exception_StateGone + */ + protected function _acquireStateRowLock() + { + $collection = $this->_stateCollection(); + $baseQuery = $this->_stateRowLockQuery(); + + try { + $exists = $collection->count($baseQuery); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + + if (!$exists) { + $this->_logger->warn(sprintf( + 'Could not find state for synckey %s.', + $this->_syncKey + )); + throw new Horde_ActiveSync_Exception_StateGone(); + } + + $maxWait = 100; + $forceStale = false; + $token = null; + + /* Spin up to ~10s (100 iterations * 100ms) waiting for an unlocked or + * stale document. After $maxWait iterations, force-steal the lock from + * any holder to prevent indefinite blocking. Total worst-case wait is + * ~20s (200 iterations). */ + for ($i = 0; $i < $maxWait * 2; ++$i) { + $token = time(); + $query = $baseQuery; + if ($forceStale) { + $query[self::SYNC_LOCK] = ['$exists' => true]; + } else { + $query['$or'] = [ + [self::SYNC_LOCK => ['$exists' => false]], + [self::SYNC_LOCK => ['$lt' => $token - self::STATE_ROW_LOCK_STALE_SECONDS]], + ]; + } + + try { + $results = $collection->findAndModify( + $query, + ['$set' => [self::SYNC_LOCK => $token]], + [ + self::SYNC_DATA => true, + self::SYNC_DEVID => true, + self::SYNC_MOD => true, + self::SYNC_PENDING => true, + ] + ); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + + if (!empty($results)) { + $this->_stateRowLockHeld = true; + $this->_stateLockToken = $token; + return $results; + } + + if ($i >= $maxWait) { + $forceStale = true; + } else { + usleep(100000); + } + } + + throw new Horde_ActiveSync_Exception('Could not acquire state row lock.'); + } + + /** + * Return the folder id used for collection locking. + * + * @param string|null $folderId Optional folder id override. + * + * @return string + */ + protected function _collectionLockFolderId($folderId = null) + { + if ($folderId !== null) { + return $folderId; + } + + if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { + return Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC; + } + + return !empty($this->_collection['id']) + ? $this->_collection['id'] + : Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC; + } + + /** + * Build the lock document _id for a collection. + * + * @param string $user + * @param string $devid + * @param string $folderid + * + * @return string + */ + protected function _collectionLockDocumentId($user, $devid, $folderid) + { + return $user . "\0" . $devid . "\0" . $folderid; + } + + /** + * Return [sync_user, sync_devid, sync_folderid] for collection locking. + * + * @param string|null $folderId Optional folder id override. + * + * @return array + */ + protected function _collectionLockIdentity($folderId = null) + { + if ($folderId === null && $this->_collectionLockFolderId !== null) { + $folderId = $this->_collectionLockFolderId; + } + + return [ + $this->_deviceInfo->user, + $this->_deviceInfo->id, + $this->_collectionLockFolderId($folderId), + ]; + } + + /** + * Ensure a collection lock document exists. + * + * @param string $user + * @param string $devid + * @param string $folderid + * + * @throws Horde_ActiveSync_Exception + */ + protected function _ensureCollectionLockDocument($user, $devid, $folderid) + { + $collection = $this->_collectionLockCollection(); + $id = $this->_collectionLockDocumentId($user, $devid, $folderid); + + try { + if ($collection->count([self::MONGO_ID => $id])) { + return; + } + + $collection->insert([ + self::MONGO_ID => $id, + self::SYNC_USER => $user, + self::SYNC_DEVID => $devid, + self::SYNC_FOLDERID => $folderid, + ]); + } catch (Exception $e) { + if (!$collection->count([self::MONGO_ID => $id])) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + } + } + + /** + * Acquire an exclusive collection lock. + * + * @param string|null $folderId Optional folder id override. + * + * @throws Horde_ActiveSync_Exception + */ + protected function _acquireCollectionLock($folderId = null) + { + if ($this->_collectionLockHeld) { + return; + } + + [$user, $devid, $lockFolderid] = $this->_collectionLockIdentity($folderId); + $this->_ensureCollectionLockDocument($user, $devid, $lockFolderid); + + $collection = $this->_collectionLockCollection(); + $id = $this->_collectionLockDocumentId($user, $devid, $lockFolderid); + $maxWait = 100; + $forceStale = false; + $token = null; + + for ($i = 0; $i < $maxWait * 2; ++$i) { + $token = random_int(1, PHP_INT_MAX); + $query = [self::MONGO_ID => $id]; + if ($forceStale) { + $query[self::LOCK_TOKEN] = ['$exists' => true]; + } else { + $query['$or'] = [ + [self::LOCK_TOKEN => ['$exists' => false]], + [self::LOCK_TIME => ['$lt' => $token - self::STATE_ROW_LOCK_STALE_SECONDS]], + ]; + } + + try { + $results = $collection->findAndModify( + $query, + ['$set' => [ + self::LOCK_TOKEN => $token, + self::LOCK_TIME => time(), + ]] + ); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + + if (!empty($results)) { + $this->_collectionLockHeld = true; + $this->_collectionLockToken = $token; + $this->_collectionLockFolderId = $lockFolderid; + return; + } + + if ($i >= $maxWait) { + $forceStale = true; + } else { + usleep(100000); + } + } + + throw new Horde_ActiveSync_Exception('Could not acquire collection lock.'); + } + + /** + * Release a collection lock acquired by _acquireCollectionLock(). + * + * @param boolean $commit Unused for Mongo; kept for parity with Sql. + */ + protected function _releaseCollectionLock($commit = false) + { + if (!$this->_collectionLockHeld) { + return; + } + + [$user, $devid, $lockFolderid] = $this->_collectionLockIdentity(); + $id = $this->_collectionLockDocumentId($user, $devid, $lockFolderid); + + try { + $this->_collectionLockCollection()->update( + [ + self::MONGO_ID => $id, + self::LOCK_TOKEN => $this->_collectionLockToken, + ], + ['$unset' => [ + self::LOCK_TOKEN => '', + self::LOCK_TIME => '', + ]] + ); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + } + + $this->_collectionLockHeld = false; + $this->_collectionLockToken = null; + $this->_collectionLockFolderId = null; + } + /** * Update the serverid for a given folder uid in the folder's state object. * Needed when a folder is renamed on a client, but the UID must remain the @@ -226,34 +616,42 @@ public function updateServerIdInState($uid, $serverid) ) ); - $query = [ - self::SYNC_DEVID => $this->_deviceInfo->id, - self::SYNC_USER => $this->_deviceInfo->user, - self::SYNC_FOLDERID => $uid, - ]; - + $this->_acquireCollectionLock($uid); try { - $cursor = $this->_db->selectCollection(self::COLLECTION_STATE) - ->find($query, [self::SYNC_DATA => true]); - } catch (Exception $e) { - $this->_logger->err($e->getMessage()); - throw new Horde_ActiveSync_Exception($e); - } + $query = [ + self::SYNC_DEVID => $this->_deviceInfo->id, + self::SYNC_USER => $this->_deviceInfo->user, + self::SYNC_FOLDERID => $uid, + ]; - foreach ($cursor as $folder) { - $folder = unserialize($folder[self::SYNC_DATA]); - $folder->setServerId($serverid); - $folder = serialize($folder); try { - $this->_db->selectCollection(self::COLLECTION_STATE)->update( - $query, - ['$set' => [self::SYNC_DATA => $folder]], - ['multiple' => true] - ); + $cursor = $this->_db->selectCollection(self::COLLECTION_STATE) + ->find($query, [self::SYNC_DATA => true]); } catch (Exception $e) { $this->_logger->err($e->getMessage()); throw new Horde_ActiveSync_Exception($e); } + + foreach ($cursor as $folder) { + $folder = unserialize($folder[self::SYNC_DATA]); + $folder->setServerId($serverid); + $folder = serialize($folder); + try { + $this->_db->selectCollection(self::COLLECTION_STATE)->update( + $query, + ['$set' => [self::SYNC_DATA => $folder]], + ['multiple' => true] + ); + } catch (Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + } + + $this->_releaseCollectionLock(true); + } catch (Throwable $e) { + $this->_releaseCollectionLock(false); + throw $e; } } @@ -264,33 +662,9 @@ public function updateServerIdInState($uid, $serverid) */ protected function _loadState() { - try { - $results = $this->_db->selectCollection(self::COLLECTION_STATE) - ->findOne( - [ - self::MONGO_ID => $this->_syncKey, - self::SYNC_FOLDERID => $this->_collection['id'], - ], - [ - self::SYNC_DATA => true, - self::SYNC_DEVID => true, - self::SYNC_MOD => true, - self::SYNC_PENDING => true, - ] - ); - } catch (Exception $e) { - $this->_logger->err('Error in loading state from DB: ' . $e->getMessage()); - throw new Horde_ActiveSync_Exception($e); - } - - if (empty($results)) { - $this->_logger->warn(sprintf( - 'Could not find state for synckey %s.', - $this->_syncKey - )); - throw new Horde_ActiveSync_Exception_StateGone(); - } + $this->_releaseStateRowLock(false); + $results = $this->_acquireStateRowLock(); $this->_loadStateFromResults($results); } @@ -315,6 +689,7 @@ protected function _loadStateFromResults(array $results) // Restore any state or pending changes $data = unserialize($results[self::SYNC_DATA]); $pending = $results[self::SYNC_PENDING]; + $this->_syncPendingBlob = $pending; if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { $this->_folder = ($data !== false) ? $data : []; @@ -325,7 +700,12 @@ protected function _loadStateFromResults(array $results) ) ); } elseif ($this->_type == Horde_ActiveSync::REQUEST_TYPE_SYNC) { - $this->_folder = $data; + $data = $this->_normalizeSyncFolderData($data); + $this->_folder = ( + $data !== false + ? $data + : $this->_createEmptySyncFolder() + ); $this->_changes = ($pending !== false) ? $pending : null; if ($this->_changes) { $this->_logger->meta( @@ -341,17 +721,29 @@ protected function _loadStateFromResults(array $results) /** * Save the current state to storage * + * @param array $options @see Horde_ActiveSync_State_Base::save() + * * @throws Horde_ActiveSync_Exception */ - public function save() + public function save(array $options = []) { + $this->_assertValidSyncFolderBeforeSave(); + // Prepare state and pending data if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { $data = (isset($this->_folder) ? serialize($this->_folder) : ''); $pending = ''; } elseif ($this->_type == Horde_ActiveSync::REQUEST_TYPE_SYNC) { - $pending = (isset($this->_changes) ? array_values($this->_changes) : ''); + if (empty($options['preservePending'])) { + $this->_finalizeInitialSyncIfComplete(); + } $data = (isset($this->_folder) ? serialize($this->_folder) : ''); + if (!empty($options['preservePending']) && $this->_syncPendingBlob !== null) { + $pending = $this->_syncPendingBlob; + } else { + $pending = (isset($this->_changes) ? array_values($this->_changes) : ''); + $this->_syncPendingBlob = $pending; + } } else { $pending = ''; $data = ''; @@ -379,22 +771,143 @@ public function save() ); try { - $this->_db->selectCollection(self::COLLECTION_STATE)->insert($document); + $this->_saveSyncStateRow($document); + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(true); + } + } catch (Throwable $e) { + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(false); + } + throw $e; + } + } + + /** + * Persist sync state to HAS_state (update, else insert). + * + * When _loadState() has acquired a document lock, the save runs as a single + * conditional update so concurrent PING/SYNC workers cannot interleave writes + * to sync_data or sync_pending for the same sync_key. + * + * @param array $document State document fields. + * + * @throws Horde_ActiveSync_Exception + */ + protected function _saveSyncStateRow(array $document) + { + $collection = $this->_stateCollection(); + $lockHeld = $this->_stateRowLockHeld; + + try { + if ($lockHeld) { + $query = array_merge( + $this->_stateRowLockQuery(), + [self::SYNC_LOCK => $this->_stateLockToken] + ); + $set = $document; + unset($set[self::MONGO_ID]); + unset($set[self::SYNC_LOCK]); + + $result = $collection->update( + $query, + [ + '$set' => $set, + '$unset' => [self::SYNC_LOCK => ''], + ] + ); + + if (empty($result['ok']) || empty($result['n'])) { + throw new Horde_ActiveSync_Exception('Error saving state.'); + } + + $this->_stateRowLockHeld = false; + $this->_stateLockToken = null; + return; + } + + try { + $collection->insert($document); + } catch (Exception $e) { + // Might exist already if the last sync attempt failed. + $this->_logger->notice( + sprintf( + 'Previous request processing for synckey %s failed to be accepted by the client, removing previous state and trying again.', + $this->_syncKey + ) + ); + try { + $collection->remove([self::MONGO_ID => $this->_syncKey]); + $collection->insert($document); + } catch (Exception $e2) { + throw new Horde_ActiveSync_Exception('Error saving state.'); + } + } + } catch (Horde_ActiveSync_Exception $e) { + if ($lockHeld) { + $this->_releaseStateRowLock(false); + } + throw $e; } catch (Exception $e) { - // Might exist already if the last sync attempt failed. - $this->_logger->notice( + if ($lockHeld) { + $this->_releaseStateRowLock(false); + } + throw new Horde_ActiveSync_Exception('Error saving state.'); + } + } + + /** + * Update the syncStamp in the collection state, outside of any other changes. + * Used to prevent extremely large differences in syncStamps for clients + * and collections that don't often have changes. + * + * @throws Horde_ActiveSync_Exception + */ + public function updateSyncStamp() + { + $updated = false; + if (($this->_thisSyncStamp - $this->_lastSyncStamp) >= self::SYNCSTAMP_UPDATE_THRESHOLD) { + $this->_logger->meta( sprintf( - 'Previous request processing for synckey %s failed to be accepted by the client, removing previous state and trying again.', - $this->_syncKey + 'Updating sync_mod value from %s to %s without changes.', + $this->_lastSyncStamp, + $this->_thisSyncStamp ) ); + + $query = array_merge( + $this->_stateRowLockQuery(), + [ + self::SYNC_MOD => $this->_lastSyncStamp, + ] + ); + if ($this->_stateRowLockHeld) { + $query[self::SYNC_LOCK] = $this->_stateLockToken; + } + try { - $this->_db->selectCollection(self::COLLECTION_STATE)->remove([self::MONGO_ID => $this->_syncKey]); - $this->_db->selectCollection(self::COLLECTION_STATE)->insert($document); + $result = $this->_stateCollection()->update( + $query, + ['$set' => [self::SYNC_MOD => $this->_thisSyncStamp]] + ); + $updated = !empty($result['ok']) && !empty($result['n']); } catch (Exception $e) { - throw new Horde_ActiveSync_Exception('Error saving state.'); + if ($this->_stateRowLockHeld) { + $this->_releaseStateRowLock(false); + } + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(false); + } + throw new Horde_ActiveSync_Exception($e); } } + + if ($this->_stateRowLockHeld) { + $this->_releaseStateRowLock($updated); + } + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock($updated); + } } /** @@ -534,7 +1047,7 @@ public function updateState( // may be sent. We need to store the leftovers for sending next // request. foreach ($this->_changes as $key => $value) { - if ($value['id'] == $change['id']) { + if ((is_array($value) && $value['id'] == $change['id']) || $value == $change['id']) { if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { foreach ($this->_folder as $fi => $state) { if ($state['id'] == $value['id']) { @@ -559,6 +1072,7 @@ public function updateState( } } unset($this->_changes[$key]); + $this->_acknowledgeExportedChange($type, $change); break; } } diff --git a/lib/Horde/ActiveSync/State/Sql.php b/lib/Horde/ActiveSync/State/Sql.php index b6eaa728..2714dfe2 100644 --- a/lib/Horde/ActiveSync/State/Sql.php +++ b/lib/Horde/ActiveSync/State/Sql.php @@ -20,7 +20,8 @@ * sync_key: - The syncKey for the last sync * sync_pending: - If the last sync resulted in a MOREAVAILABLE, this * contains a list of UIDs that still need to be sent to - * the client. + * the client. Used by SYNC only; PING must ignore this + * column and poll IMAP STATUS instead (see getChanges()). * sync_data: - Any state data that we need to track for the specific * syncKey. Data such as current folder list on the client * (for a FOLDERSYNC) and IMAP email UIDs (for Email @@ -121,6 +122,56 @@ class Horde_ActiveSync_State_Sql extends Horde_ActiveSync_State_Base */ public const SYNCSTAMP_UPDATE_THRESHOLD = 30000; + /** + * Table used to serialize state access per device/user/folder collection. + */ + protected const COLLECTION_LOCK_TABLE = 'horde_activesync_collection_lock'; + + /** + * True while a SELECT ... FOR UPDATE row lock is held for the loaded + * sync_key (released on save() or updateSyncStamp()). + * + * @var boolean + */ + protected $_stateRowLockHeld = false; + + /** + * True if this instance started the transaction that holds the row lock. + * + * @var boolean + */ + protected $_stateRowLockTxnOwner = false; + + /** + * True while a collection lock is held (released on save(), + * updateSyncStamp(), or updateServerIdInState()). + * + * @var boolean + */ + protected $_collectionLockHeld = false; + + /** + * Token set when this instance acquired the collection lock. + * + * @var integer|null + */ + protected $_collectionLockToken = null; + + /** + * True if this instance started the transaction that holds the collection + * lock. + * + * @var boolean + */ + protected $_collectionLockTxnOwner = false; + + /** + * Folder id used for the active collection lock, if any. + * + * @var string|null + */ + protected $_collectionLockFolderId = null; + /** * Const'r * @@ -146,6 +197,331 @@ public function __construct(array $params = []) $this->_db = $params['db']; } + /** + * Roll back any row lock left open when the request ends without save(). + */ + public function __destruct() + { + if ($this->_stateRowLockHeld) { + $this->_releaseStateRowLock(false); + } + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(false); + } + } + + /** + * Release a row lock opened by _loadState(). + * + * @param boolean $commit Commit (true) or roll back (false) the lock + * transaction when this instance owns it. + */ + protected function _releaseStateRowLock($commit = false) + { + if (!$this->_stateRowLockHeld) { + return; + } + + if ($this->_stateRowLockTxnOwner) { + try { + if ($commit) { + $this->_db->commitDbTransaction(); + } else { + $this->_db->rollbackDbTransaction(); + } + } catch (Horde_Db_Exception $e) { + $this->_logger->err($e->getMessage()); + } + } + + $this->_stateRowLockHeld = false; + $this->_stateRowLockTxnOwner = false; + } + + /** + * Begin a transaction and lock the current sync state row for update. + * + * @throws Horde_ActiveSync_Exception + */ + protected function _acquireStateRowLock() + { + $started = false; + if (!$this->_db->transactionStarted()) { + $this->_db->beginDbTransaction(); + $started = true; + } + + $sql = 'SELECT sync_data, sync_devid, sync_mod, sync_pending FROM ' + . $this->_syncStateTable . ' WHERE sync_key = ?'; + $values = [$this->_syncKey]; + if (!empty($this->_collection['id'])) { + $sql .= ' AND sync_folderid = ?'; + $values[] = $this->_collection['id']; + } + /* addLock() appends FOR UPDATE on MySQL/PostgreSQL but is a no-op on + * SQLite (Horde_Db_Adapter_Pdo_Sqlite). SQLite's file-level locking + * serializes writes anyway, so the row lock is advisory only there. */ + $this->_db->addLock($sql); + + try { + $results = $this->_db->selectOne($sql, $values); + } catch (Horde_Db_Exception $e) { + if ($started) { + try { + $this->_db->rollbackDbTransaction(); + } catch (Horde_Db_Exception $e2) { + } + } + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + + if (empty($results)) { + if ($started) { + try { + $this->_db->rollbackDbTransaction(); + } catch (Horde_Db_Exception $e2) { + } + } + $this->_logger->warn( + sprintf( + 'STATE: Could not find state for synckey %s.', + $this->_syncKey + ) + ); + throw new Horde_ActiveSync_Exception_StateGone(); + } + + $this->_stateRowLockHeld = true; + $this->_stateRowLockTxnOwner = $started; + + return $results; + } + + /** + * Return the folder id used for collection locking. + * + * @param string|null $folderId Optional folder id override. + * + * @return string + */ + protected function _collectionLockFolderId($folderId = null) + { + if ($folderId !== null) { + return $folderId; + } + + if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { + return Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC; + } + + return !empty($this->_collection['id']) + ? $this->_collection['id'] + : Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC; + } + + /** + * Return [sync_user, sync_devid, sync_folderid] for collection locking. + * + * @param string|null $folderId Optional folder id override. + * + * @return array + */ + protected function _collectionLockIdentity($folderId = null) + { + if ($folderId === null && $this->_collectionLockFolderId !== null) { + $folderId = $this->_collectionLockFolderId; + } + + return [ + $this->_deviceInfo->user, + $this->_deviceInfo->id, + $this->_collectionLockFolderId($folderId), + ]; + } + + /** + * True when collection locking should not fail the request (SQLite). + * + * @return boolean + */ + protected function _collectionLockIsBestEffort() + { + return $this->_db instanceof Horde_Db_Adapter_Pdo_Sqlite; + } + + /** + * Ensure a collection lock row exists. + * + * @param string $user + * @param string $devid + * @param string $folderid + * + * @throws Horde_Db_Exception + */ + protected function _ensureCollectionLockRow($user, $devid, $folderid) + { + if ($this->_db->selectValue( + 'SELECT 1 FROM ' . self::COLLECTION_LOCK_TABLE + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?', + [$user, $devid, $folderid] + )) { + return; + } + + try { + $this->_db->insert( + 'INSERT INTO ' . self::COLLECTION_LOCK_TABLE + . ' (sync_user, sync_devid, sync_folderid, lock_token, lock_time)' + . ' VALUES (?, ?, ?, NULL, NULL)', + [$user, $devid, $folderid] + ); + } catch (Horde_Db_Exception $e) { + if ($this->_db->selectValue( + 'SELECT 1 FROM ' . self::COLLECTION_LOCK_TABLE + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?', + [$user, $devid, $folderid] + )) { + return; + } + + throw $e; + } + } + + /** + * Acquire an exclusive collection lock using portable SQL only. + * + * @param string|null $folderId Optional folder id override. + * + * @throws Horde_ActiveSync_Exception + */ + protected function _acquireCollectionLock($folderId = null) + { + if ($this->_collectionLockHeld) { + return; + } + + if (!in_array(self::COLLECTION_LOCK_TABLE, $this->_db->tables())) { + return; + } + + $started = false; + try { + [$user, $devid, $lockFolderid] = $this->_collectionLockIdentity($folderId); + if (!$this->_db->transactionStarted()) { + $this->_db->beginDbTransaction(); + $started = true; + } + + $this->_ensureCollectionLockRow($user, $devid, $lockFolderid); + + $sql = 'SELECT lock_token, lock_time FROM ' . self::COLLECTION_LOCK_TABLE + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?'; + $values = [$user, $devid, $lockFolderid]; + $this->_db->addLock($sql); + + $row = $this->_db->selectOne($sql, $values); + if (empty($row)) { + if ($started) { + $this->_db->rollbackDbTransaction(); + } + throw new Horde_ActiveSync_Exception('Collection lock row missing.'); + } + + $now = time(); + $staleBefore = $now - self::STATE_ROW_LOCK_STALE_SECONDS; + $held = ($row['lock_token'] !== null && $row['lock_token'] !== ''); + $stale = $held && ((int) $row['lock_time'] < $staleBefore); + + if ($held && !$stale) { + if ($started) { + $this->_db->rollbackDbTransaction(); + } + throw new Horde_ActiveSync_Exception('Collection lock held.'); + } + + $token = random_int(1, PHP_INT_MAX); + $this->_db->update( + 'UPDATE ' . self::COLLECTION_LOCK_TABLE + . ' SET lock_token = ?, lock_time = ?' + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?', + [$token, $now, $user, $devid, $lockFolderid] + ); + + $this->_collectionLockHeld = true; + $this->_collectionLockToken = $token; + $this->_collectionLockTxnOwner = $started; + $this->_collectionLockFolderId = $lockFolderid; + } catch (Throwable $e) { + if ($started) { + try { + $this->_db->rollbackDbTransaction(); + } catch (Horde_Db_Exception $e2) { + } + $this->_collectionLockTxnOwner = false; + } + + if ($this->_collectionLockIsBestEffort()) { + $this->_collectionLockHeld = false; + $this->_collectionLockToken = null; + $this->_collectionLockFolderId = null; + $this->_logger->meta( + 'STATE: Collection lock skipped on SQLite: ' . $e->getMessage() + ); + return; + } + + throw ($e instanceof Horde_ActiveSync_Exception) + ? $e + : new Horde_ActiveSync_Exception($e); + } + } + + /** + * Release a collection lock acquired by _acquireCollectionLock(). + * + * @param boolean $commit Commit (true) or roll back (false) the lock + * transaction when this instance owns it. + */ + protected function _releaseCollectionLock($commit = false) + { + if (!$this->_collectionLockHeld) { + return; + } + + [$user, $devid, $lockFolderid] = $this->_collectionLockIdentity(); + + try { + $this->_db->update( + 'UPDATE ' . self::COLLECTION_LOCK_TABLE + . ' SET lock_token = NULL, lock_time = NULL' + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?' + . ' AND lock_token = ?', + [$user, $devid, $lockFolderid, $this->_collectionLockToken] + ); + } catch (Horde_Db_Exception $e) { + $this->_logger->err($e->getMessage()); + } + + $this->_collectionLockHeld = false; + $this->_collectionLockToken = null; + $this->_collectionLockFolderId = null; + + if ($this->_collectionLockTxnOwner) { + try { + if ($commit) { + $this->_db->commitDbTransaction(); + } else { + $this->_db->rollbackDbTransaction(); + } + } catch (Horde_Db_Exception $e) { + $this->_logger->err($e->getMessage()); + } + $this->_collectionLockTxnOwner = false; + } + } + /** * Unserialize stored PHP data without emitting E_WARNING on failure. * @@ -191,54 +567,62 @@ public function updateServerIdInState($uid, $serverid) $uid ) ); - $sql = 'SELECT sync_key, sync_data FROM ' . $this->_syncStateTable . ' WHERE ' - . 'sync_devid = ? AND sync_user = ? AND sync_folderid = ?'; - - try { - $results = $this->_db->select( - $sql, - [$this->_deviceInfo->id, $this->_deviceInfo->user, $uid] - ); - } catch (Horde_Db_Exception $e) { - $this->_logger->err($e->getMessage()); - throw new Horde_ActiveSync_Exception($e); - } + $this->_acquireCollectionLock($uid); try { - $columns = $this->_db->columns($this->_syncStateTable); - } catch (Horde_Db_Exception $e) { - $this->_logger->err($e->getMessage()); - throw new Horde_ActiveSync_Exception($e); - } - - - $update = 'UPDATE ' . $this->_syncStateTable . ' SET sync_data = ? WHERE ' - . 'sync_devid = ? AND sync_user = ? AND sync_folderid = ? AND sync_key = ?'; + $sql = 'SELECT sync_key, sync_data FROM ' . $this->_syncStateTable . ' WHERE ' + . 'sync_devid = ? AND sync_user = ? AND sync_folderid = ?'; - foreach ($results as $result) { - $folder = $this->_unserializeState( - $columns['sync_data']->binaryToString($result['sync_data']) - ); - if ($folder === false) { - continue; - } - $folder->setServerId($serverid); - $folder = serialize($folder); try { - $this->_db->update( - $update, - [ - new Horde_Db_Value_Binary($folder), - $this->_deviceInfo->id, - $this->_deviceInfo->user, - $uid, - $result['sync_key'], - ] + $results = $this->_db->select( + $sql, + [$this->_deviceInfo->id, $this->_deviceInfo->user, $uid] ); } catch (Horde_Db_Exception $e) { $this->_logger->err($e->getMessage()); throw new Horde_ActiveSync_Exception($e); } + + try { + $columns = $this->_db->columns($this->_syncStateTable); + } catch (Horde_Db_Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + + $update = 'UPDATE ' . $this->_syncStateTable . ' SET sync_data = ? WHERE ' + . 'sync_devid = ? AND sync_user = ? AND sync_folderid = ? AND sync_key = ?'; + + foreach ($results as $result) { + $folder = $this->_unserializeState( + $columns['sync_data']->binaryToString($result['sync_data']) + ); + if ($folder === false) { + continue; + } + $folder->setServerId($serverid); + $folder = serialize($folder); + try { + $this->_db->update( + $update, + [ + new Horde_Db_Value_Binary($folder), + $this->_deviceInfo->id, + $this->_deviceInfo->user, + $uid, + $result['sync_key'], + ] + ); + } catch (Horde_Db_Exception $e) { + $this->_logger->err($e->getMessage()); + throw new Horde_ActiveSync_Exception($e); + } + } + + $this->_releaseCollectionLock(true); + } catch (Throwable $e) { + $this->_releaseCollectionLock(false); + throw $e; } } @@ -249,31 +633,9 @@ public function updateServerIdInState($uid, $serverid) */ protected function _loadState() { - // Load the previous syncState from storage - $sql = 'SELECT sync_data, sync_devid, sync_mod, sync_pending FROM ' - . $this->_syncStateTable . ' WHERE sync_key = ?'; - $values = [$this->_syncKey]; - if (!empty($this->_collection['id'])) { - $sql .= ' AND sync_folderid = ?'; - $values[] = $this->_collection['id']; - } - try { - $results = $this->_db->selectOne($sql, $values); - } catch (Horde_Db_Exception $e) { - $this->_logger->err($e->getMessage()); - throw new Horde_ActiveSync_Exception($e); - } - - if (empty($results)) { - $this->_logger->warn( - sprintf( - 'STATE: Could not find state for synckey %s.', - $this->_syncKey - ) - ); - throw new Horde_ActiveSync_Exception_StateGone(); - } + $this->_releaseStateRowLock(false); + $results = $this->_acquireStateRowLock(); $this->_loadStateFromResults($results); } @@ -300,10 +662,13 @@ protected function _loadStateFromResults($results) $this->_logger->err($e->getMessage()); throw new Horde_ActiveSync_Exception($e); } - $data = $this->_unserializeState( - $columns['sync_data']->binaryToString($results['sync_data']) - ); - $pending = $this->_unserializeState($results['sync_pending']); + $rawSyncData = $columns['sync_data']->binaryToString($results['sync_data']); + $data = $this->_unserializeState($rawSyncData); + $rawSyncPending = !empty($results['sync_pending']) + ? $columns['sync_pending']->binaryToString($results['sync_pending']) + : ''; + $this->_syncPendingBlob = ($rawSyncPending !== '') ? $rawSyncPending : null; + $pending = $this->_unserializeState($rawSyncPending); if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { $this->_folder = ($data !== false) ? $data : []; @@ -314,14 +679,11 @@ protected function _loadStateFromResults($results) ) ); } elseif ($this->_type == Horde_ActiveSync::REQUEST_TYPE_SYNC) { - // @TODO: This shouldn't default to an empty folder object, - // if we don't have the data, it's an exception. + $data = $this->_normalizeSyncFolderData($data, $rawSyncData); $this->_folder = ( $data !== false ? $data - : ($this->_collection['class'] == Horde_ActiveSync::CLASS_EMAIL - ? new Horde_ActiveSync_Folder_Imap($this->_collection['serverid'], Horde_ActiveSync::CLASS_EMAIL) - : new Horde_ActiveSync_Folder_Collection($this->_collection['serverid'], $this->_collection['class'])) + : $this->_createEmptySyncFolder() ); $this->_changes = ($pending !== false) ? $pending : null; if ($this->_changes) { @@ -345,6 +707,7 @@ protected function _loadStateFromResults($results) */ public function updateSyncStamp() { + $updated = false; if (($this->_thisSyncStamp - $this->_lastSyncStamp) >= self::SYNCSTAMP_UPDATE_THRESHOLD) { $this->_logger->meta( sprintf( @@ -356,7 +719,7 @@ public function updateSyncStamp() $sql = 'UPDATE ' . $this->_syncStateTable . ' SET sync_mod = ?' . ' WHERE sync_mod = ? AND sync_key = ? AND sync_user = ? AND sync_folderid = ?'; try { - $this->_db->update( + $updated = (bool) $this->_db->update( $sql, [ $this->_thisSyncStamp, @@ -367,40 +730,90 @@ public function updateSyncStamp() ] ); } catch (Horde_Db_Exception $e) { + $this->_releaseStateRowLock(false); + $this->_releaseCollectionLock(false); throw new Horde_ActiveSync_Exception($e); } } + + if ($this->_stateRowLockHeld) { + $this->_releaseStateRowLock($updated); + } + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock($updated); + } } /** * Save the current state to storage * + * @param array $options @see Horde_ActiveSync_State_Base::save() + * + * @throws Horde_ActiveSync_Exception + */ + public function save(array $options = []) + { + try { + $this->_saveState($options); + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(true); + } + } catch (Throwable $e) { + if ($this->_collectionLockHeld) { + $this->_releaseCollectionLock(false); + } + throw $e; + } + } + + /** + * @param array $options @see Horde_ActiveSync_State_Base::save() + * * @throws Horde_ActiveSync_Exception */ - public function save() + protected function _saveState(array $options = []) { + $this->_assertValidSyncFolderBeforeSave(); + // Prepare state and pending data if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC) { $data = (isset($this->_folder) ? serialize($this->_folder) : ''); $pending = ''; } elseif ($this->_type == Horde_ActiveSync::REQUEST_TYPE_SYNC) { - $pending = (isset($this->_changes) ? serialize(array_values($this->_changes)) : ''); + if (empty($options['preservePending'])) { + $this->_finalizeInitialSyncIfComplete(); + } $data = (isset($this->_folder) ? serialize($this->_folder) : ''); + if (!empty($options['preservePending']) && $this->_syncPendingBlob !== null) { + $pendingString = (string) $this->_syncPendingBlob; + } else { + $pendingString = (isset($this->_changes) + ? serialize(array_values($this->_changes)) + : ''); + if ($pendingString !== '') { + $this->_syncPendingBlob = $pendingString; + } + } } else { $pending = ''; $data = ''; } + if ($this->_type == Horde_ActiveSync::REQUEST_TYPE_SYNC) { + $this->_assertSyncDataBlob($data); + } + // If we are setting the first synckey iteration, do not save the // syncstamp/mod, otherwise we will never get the initial set of data. + $pendingString = isset($pendingString) ? $pendingString : ''; $params = [ 'sync_key' => $this->_syncKey, - 'sync_data' => new Horde_Db_Value_Binary($data), + 'sync_data' => new Horde_Db_Value_Binary((string) $data), 'sync_devid' => $this->_deviceInfo->id, 'sync_mod' => (self::getSyncKeyCounter($this->_syncKey) == 1 ? 0 : $this->_thisSyncStamp), 'sync_folderid' => (!empty($this->_collection['id']) ? $this->_collection['id'] : Horde_ActiveSync::REQUEST_TYPE_FOLDERSYNC), 'sync_user' => $this->_deviceInfo->user, - 'sync_pending' => $pending, + 'sync_pending' => $pendingString, 'sync_timestamp' => time(), ]; $this->_logger->meta( @@ -408,12 +821,13 @@ public function save() 'STATE: Saving state: %s', serialize([ $params['sync_key'], - $params['sync_data'], + strlen($data), $params['sync_devid'], $params['sync_mod'], $params['sync_folderid'], $params['sync_user'], $this->_changes ? count($this->_changes) : 0, + strlen($pendingString), time()]) ) ); @@ -423,9 +837,9 @@ public function save() /** * Persist sync state using Horde_Db (update, else insert). * - * Avoids DELETE+INSERT and database-specific UPSERT syntax. Concurrent - * PING/SYNC requests for the same sync_key may race; a failed insert is - * retried as an update when the row already exists. + * When _loadState() has acquired a row lock, the save runs in that same + * transaction so concurrent PING/SYNC workers cannot interleave writes to + * sync_data or sync_pending for the same sync_key. * * @param array $params Column data for horde_activesync_state. * @@ -433,17 +847,29 @@ public function save() */ protected function _saveSyncStateRow(array $params) { - $where = ['sync_key = ?', [$params['sync_key']]]; + $where = [ + 'sync_key = ? AND sync_folderid = ? AND sync_devid = ? AND sync_user = ?', + [ + $params['sync_key'], + $params['sync_folderid'], + $params['sync_devid'], + $params['sync_user'], + ], + ]; $started = false; + $lockHeld = $this->_stateRowLockHeld; - if (!$this->_db->transactionStarted()) { + if (!$lockHeld && !$this->_db->transactionStarted()) { $this->_db->beginDbTransaction(); $started = true; } try { - if ($this->_db->updateBlob($this->_syncStateTable, $params, $where)) { - if ($started) { + $updated = $this->_db->updateBlob($this->_syncStateTable, $params, $where); + if ($updated) { + if ($lockHeld) { + $this->_releaseStateRowLock(true); + } elseif ($started) { $this->_db->commitDbTransaction(); } return; @@ -458,27 +884,38 @@ protected function _saveSyncStateRow(array $params) ); } catch (Horde_Db_Exception $e) { if (!$this->_syncStateExists($params['sync_key'])) { + $this->_logger->err( + sprintf( + 'STATE: Persist INSERT failed for synckey %s folder %s: %s', + $params['sync_key'], + $params['sync_folderid'], + $e->getMessage() + ) + ); throw $e; } - // TODO: Switch to DI PSR-3 Logger - Horde::log( - 'STATE: Concurrent insert for synckey ' + $message = 'STATE: Concurrent insert for synckey ' . $params['sync_key'] - . '; updating existing row.', - 'DEBUG' - ); + . '; updating existing row.'; + Horde::log($message, 'DEBUG'); + $this->_logger->meta($message); - if (!$this->_db->updateBlob($this->_syncStateTable, $params, $where)) { + $updated = $this->_db->updateBlob($this->_syncStateTable, $params, $where); + if (!$updated) { throw $e; } } - if ($started) { + if ($lockHeld) { + $this->_releaseStateRowLock(true); + } elseif ($started) { $this->_db->commitDbTransaction(); } } catch (Horde_Db_Exception $e) { - if ($started) { + if ($lockHeld) { + $this->_releaseStateRowLock(false); + } elseif ($started) { try { $this->_db->rollbackDbTransaction(); } catch (Horde_Db_Exception $e2) { @@ -695,6 +1132,7 @@ public function updateState( } } unset($this->_changes[$key]); + $this->_acknowledgeExportedChange($type, $change); break; } } diff --git a/migration/Horde/ActiveSync/24_horde_activesync_addcollectionlock.php b/migration/Horde/ActiveSync/24_horde_activesync_addcollectionlock.php new file mode 100644 index 00000000..5f1f3c8e --- /dev/null +++ b/migration/Horde/ActiveSync/24_horde_activesync_addcollectionlock.php @@ -0,0 +1,26 @@ +tables())) { + return; + } + + $t = $this->createTable('horde_activesync_collection_lock', ['autoincrementKey' => false]); + $t->column('sync_user', 'string', ['limit' => 255, 'null' => false]); + $t->column('sync_devid', 'string', ['limit' => 255, 'null' => false]); + $t->column('sync_folderid', 'string', ['limit' => 255, 'null' => false]); + $t->column('lock_token', 'bigint'); + $t->column('lock_time', 'integer'); + $t->primaryKey(['sync_user', 'sync_devid', 'sync_folderid']); + $t->end(); + } + + public function down() + { + $this->dropTable('horde_activesync_collection_lock'); + } + +} diff --git a/test/Horde/ActiveSync/ImapAdapterTest.php b/test/Horde/ActiveSync/ImapAdapterTest.php index b62937b4..f6efd922 100644 --- a/test/Horde/ActiveSync/ImapAdapterTest.php +++ b/test/Horde/ActiveSync/ImapAdapterTest.php @@ -12,6 +12,9 @@ use PHPUnit\Framework\Attributes\CoversNothing; use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_Folder_Imap; +use Horde_ActiveSync_Imap_Adapter; /** * @coversNothing @@ -19,6 +22,43 @@ #[CoversNothing] class ImapAdapterTest extends TestCase { + /** + * BigFamily-style: stale SYNC modseq must not re-trigger PING once the + * PING watermark has caught up. + */ + public function testPingUsesPingWatermarkNotSyncModseq() + { + $imap_client = $this->getMockBuilder('Horde_Imap_Client_Socket') + ->disableOriginalConstructor() + ->onlyMethods(['status']) + ->getMock(); + $serverStatus = [ + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 6000, + 'uidnext' => 8200, + 'messages' => 7932, + ]; + $imap_client->expects($this->once()) + ->method('status') + ->willReturn($serverStatus); + + $imap_factory = $this->_imapFactoryFixture($imap_client); + $adapter = new Horde_ActiveSync_Imap_Adapter(['factory' => $imap_factory]); + + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/BigFamily', Horde_ActiveSync::CLASS_EMAIL); + $folder->setStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 1267430887, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 8200, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 5864, + Horde_ActiveSync_Folder_Imap::MESSAGES => 7932, + ]); + $folder->updateState(); + $folder->acknowledgePingStatus($serverStatus); + + $this->assertFalse($adapter->ping($folder)); + $this->assertEquals(5864, $folder->modseq()); + $this->assertEquals(6000, $folder->pingModseq()); + } + public function testBug13711() { $this->markTestIncomplete("Useless test without all the fixtures."); @@ -28,8 +68,7 @@ public function testBug13711() ->method('fetch') ->will($this->_getFixturesFor13711()); - $imap_factory = new Horde_ActiveSync_Stub_ImapFactory(); - $imap_factory->fixture = $imap_client; + $imap_factory = $this->_imapFactoryFixture($imap_client); $adapter = new Horde_ActiveSync_Imap_Adapter(['factory' => $imap_factory]); $adapter->getMessages( @@ -48,6 +87,38 @@ public function testBug13711() ); } + protected function _imapFactoryFixture($imap_client) + { + return new class($imap_client) implements \Horde_ActiveSync_Interface_ImapFactory { + private $_imap; + + public function __construct($imap) + { + $this->_imap = $imap; + } + + public function getImapOb() + { + return $this->_imap; + } + + public function getMailboxes($force = false) + { + return []; + } + + public function getSpecialMailboxes() + { + return []; + } + + public function getMsgFlags() + { + return []; + } + }; + } + protected function _getFixturesFor13711() { $first = "TzozMToiSG9yZGVfSW1hcF9DbGllbnRfRmV0Y2hfUmVzdWx0cyI6Mzp7czo4OiIAKgBfZGF0YSI7YToxOntpOjQ2MjtPOjI4OiJIb3JkZV9JbWFwX0NsaWVudF9EYXRhX0ZldGNoIjoxOntzOjg6IgAqAF9kYXRhIjthOjY6e2k6MTQ7aToxNDtpOjEzO2k6NDYyO2k6MTA7YTowOnt9aToxO0M6MTU6IkhvcmRlX01pbWVfUGFydCI6MzA5OnthOjIwOntpOjA7aToxO2k6MTtzOjQ6InRleHQiO2k6MjtzOjg6ImNhbGVuZGFyIjtpOjM7czo0OiI4Yml0IjtpOjQ7YToxOntpOjA7czoyOiJkZSI7fWk6NTtzOjA6IiI7aTo2O3M6MDoiIjtpOjc7YToxOntzOjQ6InNpemUiO3M6NDoiMTUyMyI7fWk6ODthOjI6e3M6NzoiY2hhcnNldCI7czo1OiJVVEYtOCI7czo2OiJtZXRob2QiO3M6NzoiUkVRVUVTVCI7fWk6OTthOjA6e31pOjEwO3M6MToiMSI7aToxMTtzOjE6IgoiO2k6MTI7YTowOnt9aToxMztOO2k6MTQ7aToxNTIzO2k6MTU7TjtpOjE2O047aToxNztiOjA7aToxODtiOjA7aToxOTtOO319aTo5O0M6MzE6IkhvcmRlX0ltYXBfQ2xpZW50X0RhdGFfRW52ZWxvcGUiOjE2Mzg6e2E6Mjp7czoxOiJkIjtDOjE4OiJIb3JkZV9NaW1lX0hlYWRlcnMiOjE1Nzk6e2E6Mzp7aTowO2k6MztpOjE7YTo1OntzOjQ6IkRhdGUiO086MjM6IkhvcmRlX01pbWVfSGVhZGVyc19EYXRlIjoyOntzOjg6IgAqAF9uYW1lIjtzOjQ6IkRhdGUiO3M6MTA6IgAqAF92YWx1ZXMiO2E6MTp7aTowO3M6MzA6IkZyaSwgNyBOb3YgMjAxNCAxMzozNjo1NCArMDEwMCI7fX1zOjc6IlN1YmplY3QiO086MjY6IkhvcmRlX01pbWVfSGVhZGVyc19TdWJqZWN0IjoyOntzOjg6IgAqAF9uYW1lIjtzOjc6IlN1YmplY3QiO3M6MTA6IgAqAF92YWx1ZXMiO2E6MTp7aTowO3M6Mzk6Ik1BQyA+IEZhaHJzdHVobCA+IEJlc2NocmlmdHVuZyA+IE11c3RlciI7fX1zOjQ6ImZyb20iO086Mjg6IkhvcmRlX01pbWVfSGVhZGVyc19BZGRyZXNzZXMiOjM6e3M6MTE6ImFwcGVuZF9hZGRyIjtiOjE7czo4OiIAKgBfbmFtZSI7czo0OiJmcm9tIjtzOjEwOiIAKgBfdmFsdWVzIjtDOjIyOiJIb3JkZV9NYWlsX1JmYzgyMl9MaXN0IjoxNzQ6e2E6MTp7aTowO086MjU6IkhvcmRlX01haWxfUmZjODIyX0FkZHJlc3MiOjQ6e3M6NzoiY29tbWVudCI7YTowOnt9czo3OiJtYWlsYm94IjtzOjEyOiJtYXJpby5sb3JlbnoiO3M6ODoiACoAX2hvc3QiO3M6MTA6ImRlc2VydmUuZGUiO3M6MTI6IgAqAF9wZXJzb25hbCI7czoxMjoiTWFyaW8gTG9yZW56Ijt9fX19czoyOiJ0byI7TzoyODoiSG9yZGVfTWltZV9IZWFkZXJzX0FkZHJlc3NlcyI6Mzp7czoxMToiYXBwZW5kX2FkZHIiO2I6MTtzOjg6IgAqAF9uYW1lIjtzOjI6InRvIjtzOjEwOiIAKgBfdmFsdWVzIjtDOjIyOiJIb3JkZV9NYWlsX1JmYzgyMl9MaXN0Ijo2MDA6e2E6Mzp7aTowO086MjU6IkhvcmRlX01haWxfUmZjODIyX0FkZHJlc3MiOjQ6e3M6NzoiY29tbWVudCI7YTowOnt9czo3OiJtYWlsYm94IjtzOjE2OiJwYXRyaWNrLmxvZXNjaGVyIjtzOjg6IgAqAF9ob3N0IjtzOjEzOiJiaWxmaW5nZXIuY29tIjtzOjEyOiIAKgBfcGVyc29uYWwiO3M6NTY6IidMw7ZzY2hlciwgUGF0cmljayAoQmlsZmluZ2VyIFJlYWwgRXN0YXRlIEFyZ29uZW8gR21iSCknIjt9aToxO086MjU6IkhvcmRlX01haWxfUmZjODIyX0FkZHJlc3MiOjQ6e3M6NzoiY29tbWVudCI7YTowOnt9czo3OiJtYWlsYm94IjtzOjEzOiJncmVnb3IuYm90enVtIjtzOjg6IgAqAF9ob3N0IjtzOjEzOiJiaWxmaW5nZXIuY29tIjtzOjEyOiIAKgBfcGVyc29uYWwiO3M6NTM6IidCb3R6dW0sIEdyZWdvciAoQmlsZmluZ2VyIFJlYWwgRXN0YXRlIEFyZ29uZW8gR21iSCknIjt9aToyO086MjU6IkhvcmRlX01haWxfUmZjODIyX0FkZHJlc3MiOjQ6e3M6NzoiY29tbWVudCI7YTowOnt9czo3OiJtYWlsYm94IjtzOjQ6InJ1cHAiO3M6ODoiACoAX2hvc3QiO3M6MTE6ImluZGl0ZWMuY29tIjtzOjEyOiIAKgBfcGVyc29uYWwiO3M6MTQ6IidGbG9yaWFuIFJ1cHAnIjt9fX19czoxMDoiTWVzc2FnZS1JRCI7TzoyODoiSG9yZGVfTWltZV9IZWFkZXJzX01lc3NhZ2VpZCI6Mjp7czo4OiIAKgBfbmFtZSI7czoxMDoiTWVzc2FnZS1JRCI7czoxMDoiACoAX3ZhbHVlcyI7YToxOntpOjA7czo0NDoiPDAwOTEwMWNmZmE4NyQ4MzhlOGM0MCQ4YWFiYTRjMCRAZGVzZXJ2ZS5kZT4iO319fWk6MjtzOjE6IgoiO319czoxOiJ2IjtpOjM7fX1pOjM7YToxOntpOjA7czo3MDA6IkZyb206IE1hcmlvIExvcmVueiA8bWFyaW8ubG9yZW56QGRlc2VydmUuZGU+DQpUbzogIj0/dXRmLTg/Yj9KMHpEdG5OamFHVnlMQT09Pz0gUGF0cmljayAoQmlsZmluZ2VyIFJlYWwgRXN0YXRlIEFyZ29uZW8NCiBHbWJIKSciIDxwYXRyaWNrLmxvZXNjaGVyQGJpbGZpbmdlci5jb20+LCAiJ0JvdHp1bSwgR3JlZ29yIChCaWxmaW5nZXIgUmVhbA0KIEVzdGF0ZSBBcmdvbmVvIEdtYkgpJyIgPGdyZWdvci5ib3R6dW1AYmlsZmluZ2VyLmNvbT4sICdGbG9yaWFuIFJ1cHAnDQogPHJ1cHBAaW5kaXRlYy5jb20+DQpTdWJqZWN0OiBNQUMgPiBGYWhyc3R1aGwgPiBCZXNjaHJpZnR1bmcgPiBNdXN0ZXINCkRhdGU6IEZyaSwgNyBOb3YgMjAxNCAxMzozNjo1NCArMDEwMA0KTWVzc2FnZS1JRDogPDAwOTEwMWNmZmE4NyQ4MzhlOGM0MCQ4YWFiYTRjMCRAZGVzZXJ2ZS5kZT4NCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IHRleHQvY2FsZW5kYXI7IGNoYXJzZXQ9VVRGLTg7IG1ldGhvZD1SRVFVRVNUDQpDb250ZW50LVRyYW5zZmVyLUVuY29kaW5nOiA4Yml0DQpYLU1haWxlcjogTWljcm9zb2Z0IE91dGxvb2sgMTUuMA0KVGhyZWFkLUluZGV4OiBBYy82aDRLMDhSa0Q3UndYUnUyMkprczA1ZHNuTlFBQUFBUFENCkNvbnRlbnQtTGFuZ3VhZ2U6IGRlDQpVc2VyLUFnZW50OiBIb3JkZSBBcHBsaWNhdGlvbiBGcmFtZXdvcmsgNQ0KDQoiO319fX1zOjExOiIAKgBfa2V5VHlwZSI7aToyO3M6MTE6IgAqAF9vYkNsYXNzIjtzOjI4OiJIb3JkZV9JbWFwX0NsaWVudF9EYXRhX0ZldGNoIjt9"; diff --git a/test/Horde/ActiveSync/ImapFolderTest.php b/test/Horde/ActiveSync/ImapFolderTest.php index 58410610..36655334 100644 --- a/test/Horde/ActiveSync/ImapFolderTest.php +++ b/test/Horde/ActiveSync/ImapFolderTest.php @@ -166,5 +166,197 @@ public function testSerializationWithoutImapCompression() } + public function testPingCheckpointPreservesSyncModseq() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Junk', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 105, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + Horde_ActiveSync_Folder_Imap::MESSAGES => 5, + ]; + $folder->setStatus($status); + $folder->updateState(); + + $serverStatus = $status; + $serverStatus[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 250; + $folder->acknowledgePingStatus($serverStatus); + + $this->assertEquals(200, $folder->modseq()); + $this->assertEquals(250, $folder->pingModseq()); + } + + public function testPingCheckpointSerialization() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Junk', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 105, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + Horde_ActiveSync_Folder_Imap::MESSAGES => 5, + ]; + $folder->setStatus($status); + $folder->updateState(); + + $serverStatus = $status; + $serverStatus[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 250; + $folder->acknowledgePingStatus($serverStatus); + + $restored = unserialize(serialize($folder)); + $this->assertEquals(200, $restored->modseq()); + $this->assertEquals(250, $restored->pingModseq()); + } + + public function testPingCheckpointSyncedAfterSuccessfulSync() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 105, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + Horde_ActiveSync_Folder_Imap::MESSAGES => 5, + ]; + $folder->setStatus($status); + $folder->updateState(); + + $serverStatus = $status; + $serverStatus[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 250; + $folder->acknowledgePingStatus($serverStatus); + $this->assertEquals(250, $folder->pingModseq()); + $this->assertEquals(200, $folder->modseq()); + + $status[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 250; + $folder->setStatus($status); + $folder->updateState(); + + $this->assertEquals(250, $folder->modseq()); + $this->assertEquals(250, $folder->pingModseq()); + + // Partial SYNC must not move the PING checkpoint backward. + $folder->acknowledgePingStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 110, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 300, + Horde_ActiveSync_Folder_Imap::MESSAGES => 8, + ]); + $status[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 260; + $status[Horde_ActiveSync_Folder_Imap::UIDNEXT] = 106; + $folder->setStatus($status); + $folder->updateState(); + + $this->assertEquals(260, $folder->modseq()); + $this->assertEquals(300, $folder->pingModseq()); + } + + /** + * BigFamily-style case: SYNC modseq lags while PING watermark is current. + * + * After Option A, PING uses pingModseq() so a stale SYNC modseq alone must + * not re-trigger change detection once the PING checkpoint caught up. + */ + public function testPingWatermarkCanExceedSyncModseq() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/BigFamily', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 1267430887, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 8200, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 5864, + Horde_ActiveSync_Folder_Imap::MESSAGES => 7932, + ]; + $folder->setStatus($status); + $folder->updateState(); + + $serverStatus = $status; + $serverStatus[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ] = 6000; + $folder->acknowledgePingStatus($serverStatus); + + $this->assertEquals(5864, $folder->modseq()); + $this->assertEquals(6000, $folder->pingModseq()); + $this->assertFalse($folder->pingModseq() < $serverStatus[Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ]); + } + + public function testPrimedInitialSyncDefersMessagesAndInitialFlag() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Horde', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 510, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]; + $uids = range(1, 505); + + $folder->primeFolder($uids); + $folder->setStatus($status); + $folder->updateState(); + + $this->assertFalse($folder->haveInitialSync); + $this->assertEquals([], $folder->messages()); + $this->assertEquals([], $folder->added()); + } + + public function testPrimedInitialSyncAcknowledgesExportedMessages() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Horde', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 510, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]; + $uids = range(1, 505); + + $folder->primeFolder($uids); + $folder->setStatus($status); + $folder->updateState(); + + foreach (array_slice($uids, 0, 3) as $uid) { + $folder->acknowledgeExportedMessage($uid); + } + + $this->assertEquals([1, 2, 3], $folder->messages()); + $this->assertFalse($folder->haveInitialSync); + + $folder->markInitialSyncComplete(); + $this->assertTrue($folder->haveInitialSync); + } + + public function testUnserializePreservesExplicitHaveInitialSyncFalse() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Horde', Horde_ActiveSync::CLASS_EMAIL); + $folder->setStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 510, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]); + $folder->primeFolder([1, 2, 3]); + $folder->updateState(); + $folder->acknowledgeExportedMessage(1); + $folder->acknowledgeExportedMessage(2); + + $this->assertFalse($folder->haveInitialSync); + $this->assertEquals([1, 2], $folder->messages()); + + $restored = unserialize(serialize($folder)); + $this->assertFalse($restored->haveInitialSync); + $this->assertEquals([1, 2], $restored->messages()); + } + + public function testIncrementalModseqUpdateStillCompletesInitialSync() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $status = [ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 105, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]; + $msg_changes = [100, 101, 102, 103, 104]; + + $folder->setChanges($msg_changes); + $folder->setStatus($status); + $folder->updateState(); + + $this->assertTrue($folder->haveInitialSync); + $this->assertEquals($msg_changes, $folder->messages()); + } + } diff --git a/test/Horde/ActiveSync/MessageBodyDataTest.php b/test/Horde/ActiveSync/MessageBodyDataTest.php index bb6d1ef5..879d3ed3 100644 --- a/test/Horde/ActiveSync/MessageBodyDataTest.php +++ b/test/Horde/ActiveSync/MessageBodyDataTest.php @@ -20,6 +20,53 @@ #[CoversNothing] class MessageBodyDataTest extends TestCase { + /** + * Dovecot may return no data when BODY[].SIZE is combined with BODY[] + * for deeply nested MIME parts. Ensure we retry without sizes. + */ + public function testFetchFallbackWithoutBodyPartSize() + { + $empty = new \Horde_Imap_Client_Fetch_Results(); + $fetch_data = new \Horde_Imap_Client_Data_Fetch(); + $fetch_data->setUid(1576); + $fetch_data->setBodyPart('1', 'plain text body', '8bit'); + $populated = new \Horde_Imap_Client_Fetch_Results(); + $populated[$fetch_data->getUid()] = $fetch_data; + + $imap_client = $this->getMockBuilder(\Horde_Imap_Client_Socket::class) + ->disableOriginalConstructor() + ->onlyMethods(['fetch']) + ->getMock(); + $imap_client->expects($this->exactly(2)) + ->method('fetch') + ->willReturnOnConsecutiveCalls($empty, $populated); + + $plain = new \Horde_Mime_Part(); + $plain->setType('text/plain'); + $plain->setContents('plain text body'); + $mime = new \Horde_ActiveSync_Mime($plain); + + $mbd = new \Horde_ActiveSync_Imap_MessageBodyData( + [ + 'imap' => $imap_client, + 'mime' => $mime, + 'uid' => 1576, + 'mbox' => new \Horde_Imap_Client_Mailbox('INBOX'), + ], + [ + 'protocolversion' => 16.0, + 'bodyprefs' => [ + \Horde_ActiveSync::BODYPREF_TYPE_PLAIN => [ + 'truncationsize' => 500, + ], + ], + ] + ); + + $this->assertNotEmpty($mbd->plain); + $this->assertEquals(15, $mbd->plain['size']); + } + public function testReturnProperlyTruncatedHtml() { $factory = new TestServer(); diff --git a/test/Horde/ActiveSync/MimeTest.php b/test/Horde/ActiveSync/MimeTest.php index 65419c69..4bec07d6 100644 --- a/test/Horde/ActiveSync/MimeTest.php +++ b/test/Horde/ActiveSync/MimeTest.php @@ -72,6 +72,52 @@ public function testHasAttachmentsWithAttachment() $this->assertEquals(false, $mime->hasiCalendar()); } + public function testSignedMultipartWithPgpKeysFindsBody() + { + $plain = new Horde_Mime_Part(); + $plain->setType('text/plain'); + $plain->setContents('Main body text'); + $plain->setDisposition('inline'); + + $pgp = new Horde_Mime_Part(); + $pgp->setType('application/pgp-keys'); + $pgp->setDescription('PGP Public Key'); + $pgp->setContents('-----BEGIN PGP PUBLIC KEY BLOCK-----'); + + $inner = new Horde_Mime_Part(); + $inner->setType('multipart/mixed'); + $inner->addPart($plain); + $inner->addPart($pgp); + + $sig = new Horde_Mime_Part(); + $sig->setType('application/pkcs7-signature'); + $sig->setName('smime.p7s'); + $sig->setDisposition('attachment'); + $sig->setContents('signature'); + + $signed = new Horde_Mime_Part(); + $signed->setType('multipart/signed'); + $signed->setContentTypeParameter('protocol', 'application/pkcs7-signature'); + $signed->addPart($inner); + $signed->addPart($sig); + + $footer = new Horde_Mime_Part(); + $footer->setType('text/plain'); + $footer->setContents('footer'); + + $root = new Horde_Mime_Part(); + $root->setType('multipart/mixed'); + $root->addPart($signed); + $root->addPart($footer); + $root->buildMimeIds(); + + $mime = new Horde_ActiveSync_Mime($root); + $this->assertEquals('1.1.1', $mime->findBody('plain')); + $this->assertEquals(true, $mime->isSigned()); + $this->assertEquals(false, $mime->isAttachment('1.1.2', 'application/pgp-keys')); + $this->assertEquals(true, $mime->isAttachment('2', 'text/plain')); + } + public function testReplaceMime() { $fixture = file_get_contents(__DIR__ . '/fixtures/signed_attachment.eml'); diff --git a/test/Horde/ActiveSync/StateTest/Mongo/CollectionLockTest.php b/test/Horde/ActiveSync/StateTest/Mongo/CollectionLockTest.php new file mode 100644 index 00000000..3e881108 --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/Mongo/CollectionLockTest.php @@ -0,0 +1,206 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @category Horde + * @package Horde_ActiveSync + * @subpackage UnitTests + */ + +use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_State_Mongo; + +/** + * @coversNothing + */ +class CollectionLockTest extends TestCase +{ + public function testAcquireCollectionLockUsesFindAndModify() + { + $lockCollection = new MongoCollectionTestDouble(); + $lockCollection->countResult = 1; + $lockCollection->findAndModifyResult = [ + Horde_ActiveSync_State_Mongo::MONGO_ID => "user@example.com\0device\0Ftest", + ]; + + $state = $this->_newState($lockCollection); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + + $acquire = $this->_method($state, '_acquireCollectionLock'); + $acquire->invoke($state); + + $this->assertTrue($this->_getProperty($state, '_collectionLockHeld')); + $this->assertNotNull($this->_getProperty($state, '_collectionLockToken')); + $this->assertSame(1, $lockCollection->countCalls); + $this->assertSame(1, $lockCollection->findAndModifyCalls); + } + + public function testSaveReleasesCollectionLock() + { + $lockCollection = new MongoCollectionTestDouble(); + $lockCollection->updateResult = ['ok' => 1, 'n' => 1]; + $stateCollection = new MongoCollectionTestDouble(); + $stateCollection->updateResult = ['ok' => 1, 'n' => 1]; + + $state = $this->_newState($lockCollection, $stateCollection); + $this->_primeForSave($state); + $this->_setProperty($state, '_collectionLockHeld', true); + $this->_setProperty($state, '_collectionLockToken', 4242); + $this->_setProperty($state, '_collectionLockFolderId', 'Ftest'); + + $state->save(); + + $this->assertFalse($this->_getProperty($state, '_collectionLockHeld')); + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + $this->assertSame(1, $lockCollection->updateCalls); + $this->assertArrayHasKey( + Horde_ActiveSync_State_Mongo::LOCK_TOKEN, + $lockCollection->lastUpdateQuery + ); + $this->assertSame( + [ + '$unset' => [ + Horde_ActiveSync_State_Mongo::LOCK_TOKEN => '', + Horde_ActiveSync_State_Mongo::LOCK_TIME => '', + ], + ], + $lockCollection->lastUpdateDoc + ); + } + + public function testUpdateSyncStampReleasesCollectionLock() + { + $lockCollection = new MongoCollectionTestDouble(); + $lockCollection->updateResult = ['ok' => 1, 'n' => 1]; + $stateCollection = new MongoCollectionTestDouble(); + + $state = $this->_newState($lockCollection, $stateCollection); + $this->_setProperty($state, '_syncKey', '{test-lock}3'); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + $this->_setProperty($state, '_lastSyncStamp', 10); + $this->_setProperty($state, '_thisSyncStamp', 11); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateLockToken', 12345); + $this->_setProperty($state, '_collectionLockHeld', true); + $this->_setProperty($state, '_collectionLockToken', 67890); + $this->_setProperty($state, '_collectionLockFolderId', 'Ftest'); + + $state->updateSyncStamp(); + + $this->assertFalse($this->_getProperty($state, '_collectionLockHeld')); + $this->assertSame(1, $lockCollection->updateCalls); + $this->assertSame( + 67890, + $lockCollection->lastUpdateQuery[Horde_ActiveSync_State_Mongo::LOCK_TOKEN] + ); + } + + protected function _newState( + MongoCollectionTestDouble $lockCollection, + ?MongoCollectionTestDouble $stateCollection = null + ) { + $db = new MongoDbTestDoubleWithLock( + $lockCollection, + $stateCollection ?: new MongoCollectionTestDouble() + ); + $ref = new \ReflectionClass(Horde_ActiveSync_State_Mongo::class); + $state = $ref->newInstanceWithoutConstructor(); + $this->_setProperty($state, '_db', $db); + + $device = (object) [ + 'id' => 'device', + 'user' => 'user@example.com', + ]; + + $this->_setProperty( + $state, + '_logger', + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + $this->_setProperty($state, '_deviceInfo', $device); + + return $state; + } + + protected function _primeForSave(Horde_ActiveSync_State_Mongo $state) + { + $folder = new \Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_folder', $folder); + $this->_setProperty($state, '_syncKey', '{test-lock}2'); + $this->_setProperty($state, '_collection', [ + 'id' => 'Ftest', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + ]); + $this->_setProperty($state, '_thisSyncStamp', 100); + $this->_setProperty($state, '_changes', null); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateLockToken', 99999); + } + + protected function _setProperty($object, $name, $value) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + $prop->setValue($object, $value); + } + + protected function _getProperty($object, $name) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + return $prop->getValue($object); + } + + protected function _method($object, $name) + { + $ref = new \ReflectionClass($object); + $method = $ref->getMethod($name); + $method->setAccessible(true); + return $method; + } +} + +class MongoDbTestDoubleWithLock +{ + private $_lockCollection; + private $_stateCollection; + + public function __construct( + MongoCollectionTestDouble $lockCollection, + MongoCollectionTestDouble $stateCollection + ) { + $this->_lockCollection = $lockCollection; + $this->_stateCollection = $stateCollection; + } + + public function selectCollection($name) + { + if ($name === Horde_ActiveSync_State_Mongo::COLLECTION_LOCK) { + return $this->_lockCollection; + } + + return $this->_stateCollection; + } +} + +} diff --git a/test/Horde/ActiveSync/StateTest/Mongo/RowLockTest.php b/test/Horde/ActiveSync/StateTest/Mongo/RowLockTest.php new file mode 100644 index 00000000..c75b0261 --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/Mongo/RowLockTest.php @@ -0,0 +1,222 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @category Horde + * @package Horde_ActiveSync + * @subpackage UnitTests + */ + +use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_State_Mongo; + +/** + * @coversNothing + */ +class RowLockTest extends TestCase +{ + public function testAcquireStateRowLockUsesFindAndModify() + { + $collection = new MongoCollectionTestDouble(); + $collection->countResult = 1; + $collection->findAndModifyResult = [ + Horde_ActiveSync_State_Mongo::SYNC_DATA => '', + Horde_ActiveSync_State_Mongo::SYNC_DEVID => 'device', + Horde_ActiveSync_State_Mongo::SYNC_MOD => 1, + Horde_ActiveSync_State_Mongo::SYNC_PENDING => '', + ]; + + $state = $this->_newState($collection); + $this->_setProperty($state, '_syncKey', '{test-lock}1'); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + + $acquire = $this->_method($state, '_acquireStateRowLock'); + $acquire->invoke($state); + + $this->assertTrue($this->_getProperty($state, '_stateRowLockHeld')); + $this->assertNotNull($this->_getProperty($state, '_stateLockToken')); + $this->assertSame(1, $collection->countCalls); + $this->assertSame(1, $collection->findAndModifyCalls); + } + + public function testSaveReleasesDocumentLock() + { + $collection = new MongoCollectionTestDouble(); + $collection->updateResult = ['ok' => 1, 'n' => 1]; + + $state = $this->_newState($collection); + $this->_primeForSave($state); + + $state->save(); + + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + $this->assertSame(1, $collection->updateCalls); + $this->assertArrayHasKey( + Horde_ActiveSync_State_Mongo::SYNC_LOCK, + $collection->lastUpdateQuery + ); + $this->assertArrayHasKey( + Horde_ActiveSync_State_Mongo::SYNC_LOCK, + $collection->lastUpdateDoc['$unset'] + ); + } + + public function testUpdateSyncStampReleasesUnusedLock() + { + $collection = new MongoCollectionTestDouble(); + $collection->updateResult = ['ok' => 1, 'n' => 1]; + + $state = $this->_newState($collection); + $this->_setProperty($state, '_syncKey', '{test-lock}3'); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + $this->_setProperty($state, '_lastSyncStamp', 10); + $this->_setProperty($state, '_thisSyncStamp', 11); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateLockToken', 12345); + + $state->updateSyncStamp(); + + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + $this->assertSame(1, $collection->updateCalls); + $this->assertSame( + ['$unset' => [Horde_ActiveSync_State_Mongo::SYNC_LOCK => '']], + $collection->lastUpdateDoc + ); + } + + protected function _newState(MongoCollectionTestDouble $collection) + { + $db = new MongoDbTestDouble($collection); + $ref = new \ReflectionClass(Horde_ActiveSync_State_Mongo::class); + $state = $ref->newInstanceWithoutConstructor(); + $this->_setProperty($state, '_db', $db); + + $device = $this->getMockBuilder('Horde_ActiveSync_Device') + ->disableOriginalConstructor() + ->getMock(); + $device->id = 'device'; + $device->user = 'user@example.com'; + + $this->_setProperty( + $state, + '_logger', + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + $this->_setProperty($state, '_deviceInfo', $device); + + return $state; + } + + protected function _primeForSave(Horde_ActiveSync_State_Mongo $state) + { + $folder = new \Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_folder', $folder); + $this->_setProperty($state, '_syncKey', '{test-lock}2'); + $this->_setProperty($state, '_collection', [ + 'id' => 'Ftest', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + ]); + $this->_setProperty($state, '_thisSyncStamp', 100); + $this->_setProperty($state, '_changes', null); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateLockToken', 99999); + } + + protected function _setProperty($object, $name, $value) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + $prop->setValue($object, $value); + } + + protected function _getProperty($object, $name) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + return $prop->getValue($object); + } + + protected function _method($object, $name) + { + $ref = new \ReflectionClass($object); + $method = $ref->getMethod($name); + $method->setAccessible(true); + return $method; + } +} + +class MongoCollectionTestDouble +{ + public $countResult = 0; + public $findAndModifyResult = []; + public $updateResult = ['ok' => 1, 'n' => 1]; + public $countCalls = 0; + public $findAndModifyCalls = 0; + public $updateCalls = 0; + public $lastUpdateQuery = []; + public $lastUpdateDoc = []; + + public function count($query) + { + ++$this->countCalls; + return $this->countResult; + } + + public function findAndModify($query, $update, $fields = [], $options = []) + { + ++$this->findAndModifyCalls; + return $this->findAndModifyResult; + } + + public function update($query, $update, $options = []) + { + ++$this->updateCalls; + $this->lastUpdateQuery = $query; + $this->lastUpdateDoc = $update; + return $this->updateResult; + } + + public function insert($document) + { + } + + public function remove($query) + { + } +} + +class MongoDbTestDouble +{ + private $_collection; + + public function __construct(MongoCollectionTestDouble $collection) + { + $this->_collection = $collection; + } + + public function selectCollection($name) + { + return $this->_collection; + } +} + +} diff --git a/test/Horde/ActiveSync/StateTest/Sql/CollectionLockTest.php b/test/Horde/ActiveSync/StateTest/Sql/CollectionLockTest.php new file mode 100644 index 00000000..ff404b4c --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/Sql/CollectionLockTest.php @@ -0,0 +1,160 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @category Horde + * @package Horde_ActiveSync + * @subpackage UnitTests + */ + +namespace Horde\ActiveSync\StateTest\Sql; + +use Horde\ActiveSync\Test\Helpers\DbHelper; +use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_State_Sql; + +/** + * @coversNothing + */ +class CollectionLockTest extends TestCase +{ + public function testAcquireAndReleaseCollectionLockOnSqlite() + { + if (!extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite extension is not loaded'); + } + + $migrationDir = dirname(__DIR__, 5) . '/migration/Horde/ActiveSync'; + $db = DbHelper::createSqliteDb([ + 'migrations' => [[ + 'migrationsPath' => $migrationDir, + 'schemaTableName' => 'horde_activesync_schema_info', + ]], + ]); + + $state = $this->_newState($db); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + + $acquire = $this->_method($state, '_acquireCollectionLock'); + $acquire->invoke($state); + + $this->assertTrue($this->_getProperty($state, '_collectionLockHeld')); + + $release = $this->_method($state, '_releaseCollectionLock'); + $release->invoke($state, true); + + $this->assertFalse($this->_getProperty($state, '_collectionLockHeld')); + $row = $db->selectOne( + 'SELECT lock_token, lock_time FROM horde_activesync_collection_lock' + . ' WHERE sync_user = ? AND sync_devid = ? AND sync_folderid = ?', + ['user@example.com', 'device', 'Ftest'] + ); + $this->assertNull($row['lock_token']); + $this->assertNull($row['lock_time']); + } + + public function testSaveCommitsCollectionLockTransaction() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->method('transactionStarted')->willReturn(true); + $db->expects($this->once())->method('updateBlob')->willReturn(true); + $db->expects($this->once())->method('commitDbTransaction'); + $db->expects($this->never())->method('rollbackDbTransaction'); + + $state = $this->_newState($db); + $this->_primeForSave($state); + $this->_setProperty($state, '_stateRowLockTxnOwner', false); + $this->_setProperty($state, '_collectionLockHeld', true); + $this->_setProperty($state, '_collectionLockTxnOwner', true); + $this->_setProperty($state, '_collectionLockToken', 42); + $this->_setProperty($state, '_collectionLockFolderId', 'Ftest'); + $db->expects($this->once())->method('update')->willReturn(1); + + $state->save(); + $this->assertFalse($this->_getProperty($state, '_collectionLockHeld')); + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + } + + public function testSkipsCollectionLockWhenTableMissing() + { + if (!extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite extension is not loaded'); + } + + $db = new \Horde_Db_Adapter_Pdo_Sqlite([ + 'dbname' => ':memory:', + 'charset' => 'utf-8', + ]); + + $state = $this->_newState($db); + $acquire = $this->_method($state, '_acquireCollectionLock'); + $acquire->invoke($state); + + $this->assertFalse($this->_getProperty($state, '_collectionLockHeld')); + } + + protected function _newState($db) + { + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + + $device = (object) [ + 'id' => 'device', + 'user' => 'user@example.com', + ]; + + $this->_setProperty( + $state, + '_logger', + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + $this->_setProperty($state, '_deviceInfo', $device); + + return $state; + } + + protected function _primeForSave(Horde_ActiveSync_State_Sql $state) + { + $folder = new \Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_folder', $folder); + $this->_setProperty($state, '_syncKey', '{test-lock}2'); + $this->_setProperty($state, '_collection', [ + 'id' => 'Ftest', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + ]); + $this->_setProperty($state, '_thisSyncStamp', 100); + $this->_setProperty($state, '_changes', null); + $this->_setProperty($state, '_stateRowLockHeld', true); + } + + protected function _setProperty($object, $name, $value) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + $prop->setValue($object, $value); + } + + protected function _getProperty($object, $name) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + return $prop->getValue($object); + } + + protected function _method($object, $name) + { + $ref = new \ReflectionClass($object); + $method = $ref->getMethod($name); + $method->setAccessible(true); + return $method; + } +} diff --git a/test/Horde/ActiveSync/StateTest/Sql/InitialSyncTest.php b/test/Horde/ActiveSync/StateTest/Sql/InitialSyncTest.php new file mode 100644 index 00000000..dc68845b --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/Sql/InitialSyncTest.php @@ -0,0 +1,149 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @category Horde + * @package Horde_ActiveSync + * @subpackage UnitTests + */ + +namespace Horde\ActiveSync\StateTest\Sql; + +use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_Folder_Imap; +use Horde_ActiveSync_State_Sql; +use Horde_Db_Value_Binary; + +/** + * @coversNothing + */ +class InitialSyncTest extends TestCase +{ + public function testUpdateStateAcknowledgesExportedMessage() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Horde', Horde_ActiveSync::CLASS_EMAIL); + $folder->primeFolder(range(1, 10)); + $folder->setStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 11, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]); + $folder->updateState(); + + $state = $this->_createState($folder, [ + ['id' => 506, 'type' => Horde_ActiveSync::CHANGE_TYPE_CHANGE], + ['id' => 507, 'type' => Horde_ActiveSync::CHANGE_TYPE_CHANGE], + ]); + + $state->updateState( + Horde_ActiveSync::CHANGE_TYPE_CHANGE, + ['id' => 506, 'type' => Horde_ActiveSync::CHANGE_TYPE_CHANGE] + ); + + $this->assertEquals([506], $folder->messages()); + $this->assertCount(1, $this->_getChanges($state)); + } + + public function testSaveMarksInitialSyncCompleteWhenPendingEmpty() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/Horde', Horde_ActiveSync::CLASS_EMAIL); + $folder->primeFolder(range(1, 3)); + $folder->setStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 4, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 200, + ]); + $folder->updateState(); + $folder->acknowledgeExportedMessage(1); + $folder->acknowledgeExportedMessage(2); + $folder->acknowledgeExportedMessage(3); + + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->method('transactionStarted')->willReturn(false); + $db->expects($this->once())->method('beginDbTransaction'); + $db->expects($this->once()) + ->method('updateBlob') + ->with( + $this->equalTo('horde_activesync_state'), + $this->callback(function ($params) { + $data = $params['sync_data']; + if ($data instanceof Horde_Db_Value_Binary) { + $folder = unserialize($data->value); + } else { + $folder = unserialize($data); + } + return $folder instanceof Horde_ActiveSync_Folder_Imap + && $folder->haveInitialSync; + }), + $this->anything() + ) + ->willReturn(true); + $db->expects($this->once())->method('commitDbTransaction'); + + $state = $this->_createState($folder, null, $db); + $state->save(); + } + + protected function _createState( + Horde_ActiveSync_Folder_Imap $folder, + array $changes = null, + $db = null + ) { + if ($db === null) { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + } + + $device = $this->getMockBuilder('Horde_ActiveSync_Device') + ->disableOriginalConstructor() + ->getMock(); + $device->id = 'device'; + $device->user = 'user@example.com'; + + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + $ref = new \ReflectionClass($state); + foreach ([ + '_type' => Horde_ActiveSync::REQUEST_TYPE_SYNC, + '_folder' => $folder, + '_syncKey' => '{test}1', + '_deviceInfo' => $device, + '_collection' => [ + 'id' => 'F92ed1990', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + 'serverid' => 'INBOX/Horde', + ], + '_thisSyncStamp' => 100, + '_changes' => $changes, + '_syncPendingBlob' => null, + ] as $prop => $value) { + $p = $ref->getProperty($prop); + $p->setAccessible(true); + $p->setValue($state, $value); + } + + $logger = $ref->getProperty('_logger'); + $logger->setAccessible(true); + $logger->setValue( + $state, + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + + return $state; + } + + protected function _getChanges(Horde_ActiveSync_State_Sql $state) + { + $ref = new \ReflectionClass($state); + $p = $ref->getProperty('_changes'); + $p->setAccessible(true); + + return $p->getValue($state); + } +} diff --git a/test/Horde/ActiveSync/StateTest/Sql/RowLockTest.php b/test/Horde/ActiveSync/StateTest/Sql/RowLockTest.php new file mode 100644 index 00000000..762642f1 --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/Sql/RowLockTest.php @@ -0,0 +1,148 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @category Horde + * @package Horde_ActiveSync + * @subpackage UnitTests + */ + +namespace Horde\ActiveSync\StateTest\Sql; + +use Horde_Test_Case as TestCase; +use Horde_ActiveSync; +use Horde_ActiveSync_State_Sql; + +/** + * @coversNothing + */ +class RowLockTest extends TestCase +{ + public function testAcquireStateRowLockUsesForUpdate() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->method('transactionStarted')->willReturn(false); + $db->expects($this->once())->method('beginDbTransaction'); + $db->expects($this->once())->method('addLock'); + $db->expects($this->once()) + ->method('selectOne') + ->willReturn([ + 'sync_data' => '', + 'sync_devid' => 'device', + 'sync_mod' => 1, + 'sync_pending' => '', + ]); + + $state = $this->_newState($db); + $this->_setProperty($state, '_syncKey', '{test-lock}1'); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + + $acquire = $this->_method($state, '_acquireStateRowLock'); + $acquire->invoke($state); + + $this->assertTrue($this->_getProperty($state, '_stateRowLockHeld')); + $this->assertTrue($this->_getProperty($state, '_stateRowLockTxnOwner')); + } + + public function testSaveCommitsRowLockTransaction() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->method('transactionStarted')->willReturn(true); + $db->expects($this->once())->method('updateBlob')->willReturn(true); + $db->expects($this->once())->method('commitDbTransaction'); + $db->expects($this->never())->method('rollbackDbTransaction'); + + $state = $this->_newState($db); + $this->_primeForSave($state); + + $state->save(); + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + } + + public function testUpdateSyncStampRollsBackUnusedLock() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->expects($this->never())->method('update'); + $db->expects($this->once())->method('rollbackDbTransaction'); + + $state = $this->_newState($db); + $this->_setProperty($state, '_syncKey', '{test-lock}3'); + $this->_setProperty($state, '_collection', ['id' => 'Ftest']); + $this->_setProperty($state, '_lastSyncStamp', 10); + $this->_setProperty($state, '_thisSyncStamp', 11); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateRowLockTxnOwner', true); + + $state->updateSyncStamp(); + $this->assertFalse($this->_getProperty($state, '_stateRowLockHeld')); + } + + protected function _newState($db) + { + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + + $device = $this->getMockBuilder('Horde_ActiveSync_Device') + ->disableOriginalConstructor() + ->getMock(); + $device->id = 'device'; + $device->user = 'user@example.com'; + + $this->_setProperty( + $state, + '_logger', + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + $this->_setProperty($state, '_deviceInfo', $device); + + return $state; + } + + protected function _primeForSave(Horde_ActiveSync_State_Sql $state) + { + $folder = new \Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + $this->_setProperty($state, '_type', Horde_ActiveSync::REQUEST_TYPE_SYNC); + $this->_setProperty($state, '_folder', $folder); + $this->_setProperty($state, '_syncKey', '{test-lock}2'); + $this->_setProperty($state, '_collection', [ + 'id' => 'Ftest', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + ]); + $this->_setProperty($state, '_thisSyncStamp', 100); + $this->_setProperty($state, '_changes', null); + $this->_setProperty($state, '_stateRowLockHeld', true); + $this->_setProperty($state, '_stateRowLockTxnOwner', true); + } + + protected function _setProperty($object, $name, $value) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + $prop->setValue($object, $value); + } + + protected function _getProperty($object, $name) + { + $ref = new \ReflectionClass($object); + $prop = $ref->getProperty($name); + $prop->setAccessible(true); + return $prop->getValue($object); + } + + protected function _method($object, $name) + { + $ref = new \ReflectionClass($object); + $method = $ref->getMethod($name); + $method->setAccessible(true); + return $method; + } +} diff --git a/test/Horde/ActiveSync/StateTest/StateSqlPreservePendingTest.php b/test/Horde/ActiveSync/StateTest/StateSqlPreservePendingTest.php new file mode 100644 index 00000000..c16e8f97 --- /dev/null +++ b/test/Horde/ActiveSync/StateTest/StateSqlPreservePendingTest.php @@ -0,0 +1,212 @@ + 100, 'type' => 1]]); + $folder = new Horde_ActiveSync_Folder_Imap('INBOX/BigFamily', Horde_ActiveSync::CLASS_EMAIL); + $folder->setStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 200, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 5864, + Horde_ActiveSync_Folder_Imap::MESSAGES => 50, + ]); + $folder->updateState(); + $folder->acknowledgePingStatus([ + Horde_ActiveSync_Folder_Imap::UIDVALIDITY => 100, + Horde_ActiveSync_Folder_Imap::UIDNEXT => 200, + Horde_ActiveSync_Folder_Imap::HIGHESTMODSEQ => 6000, + Horde_ActiveSync_Folder_Imap::MESSAGES => 50, + ]); + + $db = $this->_mockDbForSave(function ($params) use ($pendingBlob) { + return $params['sync_pending'] === $pendingBlob + && $params['sync_data'] instanceof Horde_Db_Value_Binary; + }); + + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + $this->_primeSyncState($state, $folder, $pendingBlob); + + $state->save(['preservePending' => true]); + } + + public function testLoadRejectsCorruptEmptyArraySyncData() + { + $state = $this->_stateForNormalizeTest(); + + $normalize = $this->_method($state, '_normalizeSyncFolderData'); + $this->expectException('Horde_ActiveSync_Exception_StaleState'); + $normalize->invoke($state, [], 'a:0:{}'); + } + + public function testLoadRejectsSyncPendingShapedSyncData() + { + $state = $this->_stateForNormalizeTest(); + $pending = serialize([['id' => 100, 'type' => 1]]); + + $normalize = $this->_method($state, '_normalizeSyncFolderData'); + $this->expectException('Horde_ActiveSync_Exception_StaleState'); + $normalize->invoke($state, unserialize($pending), $pending); + } + + public function testLoadAllowsMissingSyncDataBlob() + { + $state = $this->_stateForNormalizeTest(); + + $normalize = $this->_method($state, '_normalizeSyncFolderData'); + $this->assertFalse($normalize->invoke($state, false, '')); + } + + public function testSaveRejectsInvalidEmailFolderState() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->expects($this->never())->method('updateBlob'); + + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + $this->_primeSyncState( + $state, + new Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL), + '' + ); + + $ref = new \ReflectionClass($state); + $folder = $ref->getProperty('_folder'); + $folder->setAccessible(true); + $folder->setValue($state, []); + + $this->expectException('Horde_ActiveSync_Exception_StaleState'); + $state->save(); + } + + public function testSaveWithoutPreservePendingClearsSyncPending() + { + $folder = new Horde_ActiveSync_Folder_Imap('INBOX', Horde_ActiveSync::CLASS_EMAIL); + + $db = $this->_mockDbForSave(function ($params) { + return $params['sync_pending'] === '' + && $params['sync_data'] instanceof Horde_Db_Value_Binary; + }); + + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + $this->_primeSyncState($state, $folder, serialize([['id' => 1, 'type' => 1]])); + + $state->save(); + } + + protected function _stateForNormalizeTest() + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + + $state = new Horde_ActiveSync_State_Sql(['db' => $db]); + $ref = new \ReflectionClass($state); + foreach ([ + '_type' => Horde_ActiveSync::REQUEST_TYPE_SYNC, + '_syncKey' => '{test}99', + '_collection' => [ + 'id' => 'Fea62ac31', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + 'serverid' => 'INBOX', + ], + ] as $prop => $value) { + $p = $ref->getProperty($prop); + $p->setAccessible(true); + $p->setValue($state, $value); + } + + $logger = $ref->getProperty('_logger'); + $logger->setAccessible(true); + $logger->setValue( + $state, + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + + return $state; + } + + protected function _method($state, $name) + { + $ref = new \ReflectionClass($state); + $method = $ref->getMethod($name); + $method->setAccessible(true); + + return $method; + } + + protected function _mockDbForSave(callable $pendingCheck) + { + $db = $this->getMockBuilder('Horde_Db_Adapter') + ->disableOriginalConstructor() + ->getMock(); + $db->method('transactionStarted')->willReturn(false); + $db->expects($this->once())->method('beginDbTransaction'); + $db->expects($this->once()) + ->method('updateBlob') + ->with( + $this->equalTo('horde_activesync_state'), + $this->callback($pendingCheck), + $this->anything() + ) + ->willReturn(true); + $db->expects($this->once())->method('commitDbTransaction'); + + return $db; + } + + protected function _primeSyncState( + Horde_ActiveSync_State_Sql $state, + Horde_ActiveSync_Folder_Imap $folder, + $pendingBlob + ) { + $device = $this->getMockBuilder('Horde_ActiveSync_Device') + ->disableOriginalConstructor() + ->getMock(); + $device->id = 'device'; + $device->user = 'user@example.com'; + + $ref = new \ReflectionClass($state); + foreach ([ + '_type' => Horde_ActiveSync::REQUEST_TYPE_SYNC, + '_folder' => $folder, + '_syncKey' => '{test}1', + '_deviceInfo' => $device, + '_collection' => ['id' => 'Fea62ac31', 'class' => Horde_ActiveSync::CLASS_EMAIL], + '_thisSyncStamp' => 100, + '_changes' => null, + '_syncPendingBlob' => $pendingBlob, + ] as $prop => $value) { + $p = $ref->getProperty($prop); + $p->setAccessible(true); + $p->setValue($state, $value); + } + + $logger = $ref->getProperty('_logger'); + $logger->setAccessible(true); + $logger->setValue( + $state, + new \Horde_ActiveSync_Log_Logger(new \Horde_Log_Handler_Null()) + ); + } +}