Skip to content

Optimize propagator #62562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CarlSchwan wants to merge 3 commits into
base: master
Choose a base branch
from
+139 −78
Open

Optimize propagator #62562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1f09b91
perf: Don't update etag of folder marked as IReliableETagStorage
CarlSchwan Jul 27, 2026
d80671d
perf(trashbin): Batch propagate file deletions
CarlSchwan Jul 27, 2026
37ac045
perf(version): Batch propagator for multi-write operations
CarlSchwan Jul 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 42 additions & 20 deletions apps/files_trashbin/lib/Trashbin.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
use OCP\EventDispatcher\IEventDispatcher;
use OCP\EventDispatcher\IEventListener;
use OCP\Exceptions\AbortedEventException;
use OCP\Files\Cache\IPropagator;
use OCP\Files\Events\Node\BeforeNodeDeletedEvent;
use OCP\Files\File;
use OCP\Files\Folder;
Expand Down Expand Up @@ -937,21 +938,27 @@ protected static function deleteFiles(array $files, string $user, int|float $ava
$size = 0;

if ($availableSpace <= 0) {
foreach ($files as $file) {
if ($availableSpace <= 0 && $expiration->isExpired($file['mtime'], true)) {
$tmp = self::delete($file['name'], $user, $file['mtime']);
Server::get(LoggerInterface::class)->info(
'remove "' . $file['name'] . '" (' . $tmp . 'B) to meet the limit of trash bin size (50% of available quota) for user "{user}"',
[
'app' => 'files_trashbin',
'user' => $user,
]
);
$availableSpace += $tmp;
$size += $tmp;
} else {
break;
$propagator = self::getUserStoragePropagator($user);
$propagator?->beginBatch();
try {
foreach ($files as $file) {
if ($availableSpace <= 0 && $expiration->isExpired($file['mtime'], true)) {
$tmp = self::delete($file['name'], $user, $file['mtime']);
Server::get(LoggerInterface::class)->info(
'remove "' . $file['name'] . '" (' . $tmp . 'B) to meet the limit of trash bin size (50% of available quota) for user "{user}"',
[
'app' => 'files_trashbin',
'user' => $user,
]
);
$availableSpace += $tmp;
$size += $tmp;
} else {
break;
}
}
} finally {
$propagator?->commitBatch();
}
}
return $size;
Expand All @@ -968,10 +975,17 @@ public static function deleteExpiredFiles($files, $user) {
$expiration = Server::get(Expiration::class);
$size = 0;
$count = 0;
foreach ($files as $file) {
$timestamp = $file['mtime'];
$filename = $file['name'];
if ($expiration->isExpired($timestamp)) {

$propagator = self::getUserStoragePropagator($user);
$propagator?->beginBatch();
try {
foreach ($files as $file) {
$timestamp = $file['mtime'];
$filename = $file['name'];
if (!$expiration->isExpired($timestamp)) {
break;
}

try {
$size += self::delete($filename, $user, $timestamp);
$count++;
Expand All @@ -991,14 +1005,22 @@ public static function deleteExpiredFiles($files, $user) {
'user' => $user,
],
);
} else {
break;
}
} finally {
$propagator?->commitBatch();
}

return [$size, $count];
}

private static function getUserStoragePropagator(string $user): ?IPropagator {
try {
return Server::get(IRootFolder::class)->getUserFolder($user)->getStorage()->getPropagator();
} catch (\Exception) {
return null;
}
}

/**
* recursive copy to copy a whole directory
*
Expand Down
135 changes: 81 additions & 54 deletions apps/files_versions/lib/Storage.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\Command\IBus;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Cache\IPropagator;
use OCP\Files\FileInfo;
use OCP\Files\Folder;
use OCP\Files\IMimeTypeDetector;
Expand Down Expand Up @@ -234,6 +235,14 @@ public static function markDeletedFile($path) {
'filename' => $filename];
}

private static function getUserStoragePropagator(string $uid): ?IPropagator {
try {
return Server::get(IRootFolder::class)->getUserFolder($uid)->getStorage()->getPropagator();
} catch (\Exception) {
return null;
}
}

/**
* delete the version from the storage and cache
*
Expand Down Expand Up @@ -264,10 +273,16 @@ public static function delete($path) {

$versions = self::getVersions($uid, $filename);
if (!empty($versions)) {
foreach ($versions as $v) {
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
self::deleteVersion($view, $filename . '.v' . $v['version']);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
$propagator = self::getUserStoragePropagator($uid);
$propagator?->beginBatch();
try {
foreach ($versions as $v) {
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
self::deleteVersion($view, $filename . '.v' . $v['version']);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
}
} finally {
$propagator?->commitBatch();
}
}
}
Expand Down Expand Up @@ -619,21 +634,27 @@ public static function expireOlderThanMaxForUser($uid) {
return $version < $threshold;
});

foreach ($versions as $version) {
$internalPath = $version->getInternalPath();
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
$propagator = self::getUserStoragePropagator($uid);
$propagator?->beginBatch();
try {
foreach ($versions as $version) {
$internalPath = $version->getInternalPath();
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);

$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
if (!is_null($versionEntity)) {
$versionsMapper->delete($versionEntity);
}
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
if (!is_null($versionEntity)) {
$versionsMapper->delete($versionEntity);
}

try {
$version->delete();
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
} catch (NotPermittedException $e) {
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
try {
$version->delete();
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
} catch (NotPermittedException $e) {
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
}
}
} finally {
$propagator?->commitBatch();
}
}

Expand Down Expand Up @@ -935,47 +956,53 @@ public static function expire($filename, $uid) {
$versionsSize = $versionsSize - $sizeOfDeletedVersions;
}

foreach ($toDelete as $key => $path) {
// Make sure to cleanup version table relations as expire does not pass deleteVersion
try {
/** @var VersionsMapper $versionsMapper */
$versionsMapper = Server::get(VersionsMapper::class);
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
$pathparts = pathinfo($path);
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
continue;
$propagator = self::getUserStoragePropagator($uid);
$propagator?->beginBatch();
try {
foreach ($toDelete as $key => $path) {
// Make sure to cleanup version table relations as expire does not pass deleteVersion
try {
/** @var VersionsMapper $versionsMapper */
$versionsMapper = Server::get(VersionsMapper::class);
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
$pathparts = pathinfo($path);
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
continue;
}
$versionsMapper->delete($versionEntity);
} catch (DoesNotExistException $e) {
}
$versionsMapper->delete($versionEntity);
} catch (DoesNotExistException $e) {
}

\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
self::deleteVersion($versionsFileview, $path);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
unset($allVersions[$key]); // update array with the versions we keep
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
}
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
self::deleteVersion($versionsFileview, $path);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
unset($allVersions[$key]); // update array with the versions we keep
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
}

// Check if enough space is available after versions are rearranged.
// If not we delete the oldest versions until we meet the size limit for versions,
// but always keep the two latest versions
$numOfVersions = count($allVersions) - 2 ;
$i = 0;
// sort oldest first and make sure that we start at the first element
ksort($allVersions);
reset($allVersions);
while ($availableSpace < 0 && $i < $numOfVersions) {
$version = current($allVersions);
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
$versionsSize -= $version['size'];
$availableSpace += $version['size'];
next($allVersions);
$i++;
// Check if enough space is available after versions are rearranged.
// If not we delete the oldest versions until we meet the size limit for versions,
// but always keep the two latest versions
$numOfVersions = count($allVersions) - 2 ;
$i = 0;
// sort oldest first and make sure that we start at the first element
ksort($allVersions);
reset($allVersions);
while ($availableSpace < 0 && $i < $numOfVersions) {
$version = current($allVersions);
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
$versionsSize -= $version['size'];
$availableSpace += $version['size'];
next($allVersions);
$i++;
}
} finally {
$propagator?->commitBatch();
}

return $versionsSize; // finally return the new size of the version history
Expand Down
20 changes: 16 additions & 4 deletions lib/private/Files/Cache/Propagator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,10 @@ public function commitBatch(): void {
// Ensure rows are always locked in the same order
uasort($this->batch, static fn (array $a, array $b) => $a['hash'] <=> $b['hash']);

// storages with reliable etags maintain their own etag, so don't churn one
// here on every batched row (matches the check in propagateChange())
$reliableEtag = $this->storage->instanceOfStorage(IReliableEtagStorage::class);

try {
$this->connection->beginTransaction();

Expand All @@ -219,17 +223,21 @@ public function commitBatch(): void {
$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('fileid', $query->createParameter('fileid')));
if (!$reliableEtag) {
$query->set('etag', $query->expr()->literal(uniqid()));
}

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid')));
if (!$reliableEtag) {
$queryWithSize->set('etag', $queryWithSize->expr()->literal(uniqid()));
}

while ($row = $result->fetchAssociative()) {
$item = $this->batch[$row['path']];
Expand All @@ -250,17 +258,21 @@ public function commitBatch(): void {
$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
if (!$reliableEtag) {
$query->set('etag', $query->expr()->literal(uniqid()));
}

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash')));
if (!$reliableEtag) {
$queryWithSize->set('etag', $queryWithSize->expr()->literal(uniqid()));
}

foreach ($this->batch as $item) {
if ($item['size']) {
Expand Down
Loading