diff --git a/lib/private/SystemTag/SystemTagObjectMapper.php b/lib/private/SystemTag/SystemTagObjectMapper.php
index 2c20d1f8000ea..915ea2b651fdb 100644
--- a/lib/private/SystemTag/SystemTagObjectMapper.php
+++ b/lib/private/SystemTag/SystemTagObjectMapper.php
@@ -358,16 +358,6 @@ public function setObjectIdsForTag(string $tagId, string $objectType, array $obj
 		if (!empty($addedObjectIds)) {
 			$this->dispatcher->dispatchTyped(new TagAssignedEvent($objectType, array_map(fn ($objectId) => (string)$objectId, $addedObjectIds), [(int)$tagId]));
 		}
-
-		// Dispatch unassign events for removed object ids
-		foreach ($removedObjectIds as $objectId) {
-			$this->dispatcher->dispatch(MapperEvent::EVENT_UNASSIGN, new MapperEvent(
-				MapperEvent::EVENT_UNASSIGN,
-				$objectType,
-				(string)$objectId,
-				[(int)$tagId]
-			));
-		}
 	}
 
 	/**
diff --git a/tests/lib/SystemTag/SystemTagObjectMapperTest.php b/tests/lib/SystemTag/SystemTagObjectMapperTest.php
index b28e69b1c1c79..e9eb95a5e9390 100644
--- a/tests/lib/SystemTag/SystemTagObjectMapperTest.php
+++ b/tests/lib/SystemTag/SystemTagObjectMapperTest.php
@@ -18,6 +18,7 @@
 use OCP\SystemTag\ISystemTag;
 use OCP\SystemTag\ISystemTagManager;
 use OCP\SystemTag\ISystemTagObjectMapper;
+use OCP\SystemTag\MapperEvent;
 use OCP\SystemTag\TagAssignedEvent;
 use OCP\SystemTag\TagNotFoundException;
 use OCP\SystemTag\TagUnassignedEvent;
@@ -258,6 +259,28 @@ public function testAssignUnassignTags(): void {
 		], $tagIdMapping);
 	}
 
+	public function testSetObjectIdsForTagDispatchesUnassignEventOncePerObject(): void {
+		$unassignedObjectIds = [];
+		$this->dispatcher->expects($this->any())->method('dispatch')->willReturnCallback(
+			function (string $eventName, Event $event) use (&$unassignedObjectIds): void {
+				if ($eventName === MapperEvent::EVENT_UNASSIGN) {
+					$unassignedObjectIds[] = $event->getObjectId();
+				}
+			}
+		);
+
+		// tag1 is assigned to objects '1' and '2', keep only '2'
+		$this->tagMapper->setObjectIdsForTag((string)$this->tag1->getId(), 'testtype', ['2']);
+
+		$this->assertEquals(['1'], $unassignedObjectIds);
+
+		// same expectation when the new object list is empty
+		$unassignedObjectIds = [];
+		$this->tagMapper->setObjectIdsForTag((string)$this->tag1->getId(), 'testtype', []);
+
+		$this->assertEquals(['2'], $unassignedObjectIds);
+	}
+
 	public function testReAssignUnassignTags(): void {
 		// reassign tag1
 		$this->tagMapper->assignTags('1', 'testtype', [$this->tag1->getId()]);