Improve user experience when downloading archives

apps/dav/composer/composer/autoload_classmap.php

@@ -254,6 +254,7 @@
     'OCA\\DAV\\Connector\\Sabre\\ShareTypeList' => $baseDir . '/../lib/Connector/Sabre/ShareTypeList.php',
     'OCA\\DAV\\Connector\\Sabre\\ShareeList' => $baseDir . '/../lib/Connector/Sabre/ShareeList.php',
     'OCA\\DAV\\Connector\\Sabre\\SharesPlugin' => $baseDir . '/../lib/Connector/Sabre/SharesPlugin.php',
+    'OCA\\DAV\\Connector\\Sabre\\StreamByteCounter' => $baseDir . '/../lib/Connector/Sabre/StreamByteCounter.php',
     'OCA\\DAV\\Connector\\Sabre\\TagList' => $baseDir . '/../lib/Connector/Sabre/TagList.php',
     'OCA\\DAV\\Connector\\Sabre\\TagsPlugin' => $baseDir . '/../lib/Connector/Sabre/TagsPlugin.php',
     'OCA\\DAV\\Connector\\Sabre\\UserIdHeaderPlugin' => $baseDir . '/../lib/Connector/Sabre/UserIdHeaderPlugin.php',

apps/dav/composer/composer/autoload_static.php

@@ -269,6 +269,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Connector\\Sabre\\ShareTypeList' => __DIR__ . '/..' . '/../lib/Connector/Sabre/ShareTypeList.php',
         'OCA\\DAV\\Connector\\Sabre\\ShareeList' => __DIR__ . '/..' . '/../lib/Connector/Sabre/ShareeList.php',
         'OCA\\DAV\\Connector\\Sabre\\SharesPlugin' => __DIR__ . '/..' . '/../lib/Connector/Sabre/SharesPlugin.php',
+        'OCA\\DAV\\Connector\\Sabre\\StreamByteCounter' => __DIR__ . '/..' . '/../lib/Connector/Sabre/StreamByteCounter.php',
         'OCA\\DAV\\Connector\\Sabre\\TagList' => __DIR__ . '/..' . '/../lib/Connector/Sabre/TagList.php',
         'OCA\\DAV\\Connector\\Sabre\\TagsPlugin' => __DIR__ . '/..' . '/../lib/Connector/Sabre/TagsPlugin.php',
         'OCA\\DAV\\Connector\\Sabre\\UserIdHeaderPlugin' => __DIR__ . '/..' . '/../lib/Connector/Sabre/UserIdHeaderPlugin.php',

apps/dav/lib/Connector/Sabre/ServerFactory.php

@@ -110,6 +110,8 @@ public function createServer(
 			$this->logger,
 			$this->eventDispatcher,
 			\OCP\Server::get(IDateTimeZone::class),
+			$this->config,
+			$this->l10n,
 		));
 
 		// Some WebDAV clients do require Class 2 WebDAV support (locking), since

apps/dav/lib/Connector/Sabre/StreamByteCounter.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OCA\DAV\Connector\Sabre;
+
+/**
+ * Class to use in combination with ByteCounterFilter to keep track of how much
+ * has been read from a stream.
+ *
+ * @see ByteCounterFilter
+ */
+class StreamByteCounter {
+	public float|int $bytes = 0;
+}

apps/dav/lib/Connector/Sabre/ZipFolderPlugin.php

@@ -8,14 +8,19 @@
  */
 namespace OCA\DAV\Connector\Sabre;
 
+use Icewind\Streams\CountWrapper;
 use OC\Streamer;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Events\BeforeZipCreatedEvent;
 use OCP\Files\File as NcFile;
 use OCP\Files\Folder as NcFolder;
 use OCP\Files\Node as NcNode;
+use OCP\Files\NotPermittedException;
+use OCP\IConfig;
 use OCP\IDateTimeZone;
+use OCP\IL10N;
+use OCP\Lock\LockedException;
 use Psr\Log\LoggerInterface;
 use Sabre\DAV\Server;
 use Sabre\DAV\ServerPlugin;
@@ -37,6 +42,8 @@ class ZipFolderPlugin extends ServerPlugin {
 	 * Reference to main server object
 	 */
 	private ?Server $server = null;
+	private bool $reportMissingFiles;
+	private array $missingInfo = [];
 
 	/**
 	 * Whether handleDownload has fully streamed an archive for the current request.
@@ -49,7 +56,10 @@ public function __construct(
 		private LoggerInterface $logger,
 		private IEventDispatcher $eventDispatcher,
 		private IDateTimeZone $timezoneFactory,
+		private IConfig $config,
+		private IL10N $l10n,
 	) {
+		$this->reportMissingFiles = $this->config->getSystemValueBool('archive_report_missing_files', false);
 	}
 
 	/**
@@ -69,27 +79,67 @@ public function initialize(Server $server): void {
 
 	/**
 	 * Adding a node to the archive streamer.
-	 * This will recursively add new nodes to the stream if the node is a directory.
+	 * @return ?string an error message if an error occurred and reporting is enabled, null otherwise
+	 * @throws NotPermittedException|LockedException
 	 */
-	protected function streamNode(Streamer $streamer, NcNode $node, string $rootPath): void {
+	protected function streamNode(Streamer $streamer, NcNode $node, string $rootPath): ?string {
 		// Remove the root path from the filename to make it relative to the requested folder
 		$filename = str_replace($rootPath, '', $node->getPath());
 
 		$mtime = $node->getMTime();
+		if ($node instanceof NcFolder) {
+			$streamer->addEmptyDir($filename, $mtime);
+			return null;
+		}
+
 		if ($node instanceof NcFile) {
-			$resource = $node->fopen('rb');
-			if ($resource === false) {
-				$this->logger->info('Cannot read file for zip stream', ['filePath' => $node->getPath()]);
-				throw new \Sabre\DAV\Exception\ServiceUnavailable('Requested file can currently not be accessed.');
+			$nodeSize = $node->getSize();
+			$stream = $node->fopen('rb');
+
+			if ($stream === false) {
+				return $this->l10n->t('File could not be opened (fopen). Please check the server logs for more information.');
 			}
-			$streamer->addFileFromStream($resource, $filename, $node->getSize(), $mtime);
-		} elseif ($node instanceof NcFolder) {
-			$streamer->addEmptyDir($filename, $mtime);
-			$content = $node->getDirectoryListing();
-			foreach ($content as $subNode) {
-				$this->streamNode($streamer, $subNode, $rootPath);
+
+			$read = 0;
+			$stream = CountWrapper::wrap($stream, function (int $written) use (&$read) {
+				return $read += $written;
+			});
+
+			if ($stream === false) {
+				return $this->l10n->t('Unable to check file for consistency check');
 			}
+
+			$fileAddedToStream = $streamer->addFileFromStream($stream, $filename, $nodeSize, $mtime);
+			if (!$fileAddedToStream) {
+				return $this->l10n->t('The archive was already finalized');
+			}
+
+			return $this->logStreamErrors($stream, $filename, $nodeSize, $read);
 		}
+
+		return null;
+	}
+
+	/**
+	 * Checks whether $stream was fully streamed or if there were other issues
+	 * with the stream, logging the error if necessary.
+	 *
+	 */
+	private function logStreamErrors(mixed $stream, string $path, float|int $expectedFileSize, float|int $readFileSize): ?string {
+		$streamMetadata = stream_get_meta_data($stream);
+		if (!is_resource($stream) || get_resource_type($stream) !== 'stream') {
+			return $this->l10n->t('Resource is not a stream or is closed.');
+		}
+
+		if ($streamMetadata['timed_out'] ?? false) {
+			return $this->l10n->t('Timeout while reading from stream.');
+		}
+
+		if (!($streamMetadata['eof'] ?? true) || $readFileSize != $expectedFileSize) {
+			return $this->l10n->t('Read %d out of %d bytes from storage. This means the connection may have been closed due to a network/storage error.', [$readFileSize, $expectedFileSize]);
+		}
+
+		return null;
 	}
 
 	/**
@@ -144,7 +194,7 @@ public function handleDownload(Request $request, Response $response): ?false {
 		}
 
 		$folder = $node->getNode();
-		$event = new BeforeZipCreatedEvent($folder, $files);
+		$event = new BeforeZipCreatedEvent($folder, $files, $this->reportMissingFiles);
 		$this->eventDispatcher->dispatchTyped($event);
 		if ((!$event->isSuccessful()) || $event->getErrorMessage() !== null) {
 			$errorMessage = $event->getErrorMessage();
@@ -157,12 +207,16 @@ public function handleDownload(Request $request, Response $response): ?false {
 			throw new Forbidden($errorMessage);
 		}
 
+		// At this point either the event handlers did not block the download
+		// or they support the new mechanism that filters out nodes that are not
+		// downloadable, in either case we can use the new API to set the iterator
 		$content = empty($files) ? $folder->getDirectoryListing() : [];
 		foreach ($files as $path) {
 			$child = $node->getChild($path);
 			assert($child instanceof Node);
 			$content[] = $child->getNode();
 		}
+		$event->setNodesIterable($this->getIterableFromNodes($content));
 
 		$archiveName = $folder->getName();
 		if (count(explode('/', trim($folder->getPath(), '/'), 3)) === 2) {
@@ -176,21 +230,74 @@ public function handleDownload(Request $request, Response $response): ?false {
 			$rootPath = dirname($folder->getPath());
 		}
 
-		$streamer = new Streamer($tarRequest, -1, count($content), $this->timezoneFactory);
+		// numberOfFiles is irrelevant as size=-1 forces the use of zip64 already
+		$streamer = new Streamer($tarRequest, -1, 0, $this->timezoneFactory);
 		$streamer->sendHeaders($archiveName);
 		// For full folder downloads we also add the folder itself to the archive
 		if (empty($files)) {
 			$streamer->addEmptyDir($archiveName);
 		}
-		foreach ($content as $node) {
-			$this->streamNode($streamer, $node, $rootPath);
+
+		foreach ($event->getNodes($rootPath) as $path => [$node, $reason]) {
+			$filename = str_replace($rootPath, '', $path);
+			if ($node === null) {
+				if ($this->reportMissingFiles) {
+					$this->missingInfo[$filename] = $reason;
+				}
+				continue;
+			}
+
+			try {
+				$streamError = $this->streamNode($streamer, $node, $rootPath);
+			} catch (\Exception $e) {
+				if (!$this->reportMissingFiles) {
+					throw $e;
+				}
+
+				$logMessage = $this->l10n->t('Error while streaming the file');
+				$this->logger->error($logMessage, ['exception' => $e]);
+				$reason = $this->l10n->t('File could not be added to the archive. Please check the server logs for more information.');
+				$this->missingInfo[$filename] = $reason;
+				continue;
+			}
+
+			if ($this->reportMissingFiles && $streamError !== null) {
+				$this->missingInfo[$filename] = $streamError;
+			}
+		}
+
+		if ($this->reportMissingFiles && !empty($this->missingInfo)) {
+			$json = json_encode($this->missingInfo, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT);
+			$stream = fopen('php://temp', 'r+');
+			fwrite($stream, $json);
+			rewind($stream);
+			$streamer->addFileFromStream($stream, 'missing_files.json', (float)strlen($json), false);
 		}
 		$streamer->finalize();
 		$this->streamed = true; // archive fully streamed
 
 		return false;
 	}
 
+	/**
+	 * Given a set of nodes, produces a list of all nodes contained in them
+	 * recursively.
+	 *
+	 * @param NcNode[] $nodes
+	 * @return iterable<NcNode>
+	 */
+	private function getIterableFromNodes(array $nodes): iterable {
+		foreach ($nodes as $node) {
+			yield $node;
+
+			if ($node instanceof NcFolder) {
+				foreach ($node->getDirectoryListing() as $child) {
+					yield from $this->getIterableFromNodes([$child]);
+				}
+			}
+		}
+	}
+
 	/**
 	 * Tell sabre/dav not to trigger its own response sending logic as the handleDownload will have already sent the response
 	 */

apps/dav/lib/Server.php

@@ -95,6 +95,7 @@
 use OCP\ITagManager;
 use OCP\IURLGenerator;
 use OCP\IUserSession;
+use OCP\L10N\IFactory;
 use OCP\Mail\IEmailValidator;
 use OCP\Mail\IMailer;
 use OCP\Profiler\IProfiler;
@@ -244,6 +245,7 @@ public function __construct(
 			\OCP\Server::get(IUserSession::class)
 		));
 
+		$config = \OCP\Server::get(IConfig::class);
 		// performance improvement plugins
 		$this->server->addPlugin(new CopyEtagHeaderPlugin());
 		$this->server->addPlugin(new RequestIdHeaderPlugin(\OCP\Server::get(IRequest::class)));
@@ -256,6 +258,8 @@ public function __construct(
 			$logger,
 			$eventDispatcher,
 			\OCP\Server::get(IDateTimeZone::class),
+			$config,
+			\OCP\Server::get(IFactory::class)->get('dav'),
 		));
 		$this->server->addPlugin(\OCP\Server::get(PaginatePlugin::class));
 		$this->server->addPlugin(new PropFindPreloadNotifyPlugin());