Add offline fallback for folder enumeration and auto-retry failed uploads

Cryptomator.xcodeproj/project.pbxproj

@@ -2067,15 +2067,6 @@
 			path = Settings;
 			sourceTree = "<group>";
 		};
-		8DB472C9A9C7437C46DCAFD7 /* Hub */ = {
-			isa = PBXGroup;
-			children = (
-				0964E7C048AC1D59105CF75C /* TrustedHubHostsViewController.swift */,
-				46F99A7913E9584535D9F5FC /* TrustedHubHostsViewModel.swift */,
-			);
-			path = Hub;
-			sourceTree = "<group>";
-		};
 		743D95FA2D76EE88002D73C3 /* MicrosoftGraph */ = {
 			isa = PBXGroup;
 			children = (
@@ -2147,6 +2138,15 @@
 			path = Purchase;
 			sourceTree = "<group>";
 		};
+		8DB472C9A9C7437C46DCAFD7 /* Hub */ = {
+			isa = PBXGroup;
+			children = (
+				0964E7C048AC1D59105CF75C /* TrustedHubHostsViewController.swift */,
+				46F99A7913E9584535D9F5FC /* TrustedHubHostsViewModel.swift */,
+			);
+			path = Hub;
+			sourceTree = "<group>";
+		};
 		B3C397FD2EB10FC0001280AC /* ShareVault */ = {
 			isa = PBXGroup;
 			children = (

CryptomatorFileProvider/DB/CachedFileDBManager.swift

@@ -12,6 +12,7 @@ import GRDB
 
 protocol CachedFileManager {
 	func getLocalCachedFileInfo(for id: Int64) throws -> LocalCachedFileInfo?
+	func getLocalCachedFileInfo(forIds ids: [Int64]) throws -> [LocalCachedFileInfo?]
 	func cacheLocalFileInfo(for id: Int64, localURL: URL, lastModifiedDate: Date?) throws
 	func removeCachedFile(for id: Int64) throws
 	func clearCache() throws
@@ -26,6 +27,16 @@ extension CachedFileManager {
 		return try getLocalCachedFileInfo(for: id)
 	}
 
+	func getLocalCachedFileInfo(for itemMetadata: [ItemMetadata]) throws -> [LocalCachedFileInfo?] {
+		let ids: [Int64] = try itemMetadata.map {
+			guard let id = $0.id else {
+				throw DBManagerError.nonSavedItemMetadata
+			}
+			return id
+		}
+		return try getLocalCachedFileInfo(forIds: ids)
+	}
+
 	func removeCachedFile(for itemIdentifier: NSFileProviderItemIdentifier) throws {
 		guard let itemID = itemIdentifier.databaseValue else {
 			return
@@ -82,6 +93,14 @@ class CachedFileDBManager: CachedFileManager {
 		}
 	}
 
+	func getLocalCachedFileInfo(forIds ids: [Int64]) throws -> [LocalCachedFileInfo?] {
+		return try database.read { db in
+			let rows = try LocalCachedFileInfo.filter(keys: ids).fetchAll(db)
+			let dict = Dictionary(uniqueKeysWithValues: rows.map { ($0.correspondingItem, $0) })
+			return ids.map { dict[$0] }
+		}
+	}
+
 	func cacheLocalFileInfo(for id: Int64, localURL: URL, lastModifiedDate: Date?) throws {
 		try database.write { db in
 			try LocalCachedFileInfo(lastModifiedDate: lastModifiedDate, correspondingItem: id, localLastModifiedDate: Date(), localURL: localURL).save(db)

CryptomatorFileProvider/DB/DatabaseHelper.swift

@@ -193,6 +193,22 @@ public struct DatabaseHelper: DatabaseHelping {
 				table.add(column: "uploadStartedAt", .date)
 			}
 		}
+		migrator.registerMigration("v4") { db in
+			try db.alter(table: "itemMetadata") { table in
+				table.add(column: "lastEnumeratedAt", .date)
+			}
+			try db.execute(sql: """
+			UPDATE itemMetadata
+			SET lastEnumeratedAt = CURRENT_TIMESTAMP
+			WHERE type = 'folder'
+			  AND EXISTS (
+			    SELECT 1
+			    FROM itemMetadata AS child
+			    WHERE child.parentID = itemMetadata.id
+			      AND child.id != itemMetadata.id
+			  )
+			""")
+		}
 		try migrator.migrate(dbWriter)
 	}
 

CryptomatorFileProvider/DB/ItemMetadata.swift

@@ -31,6 +31,7 @@ public class ItemMetadata: Record, Codable {
 	var isMaybeOutdated: Bool
 	var favoriteRank: Int64?
 	var tagData: Data?
+	var lastEnumeratedAt: Date?
 
 	required init(row: Row) throws {
 		self.id = row[Columns.id]
@@ -45,14 +46,15 @@ public class ItemMetadata: Record, Codable {
 		self.isMaybeOutdated = row[Columns.isMaybeOutdated]
 		self.favoriteRank = row[Columns.favoriteRank]
 		self.tagData = row[Columns.tagData]
+		self.lastEnumeratedAt = row[Columns.lastEnumeratedAt]
 		try super.init(row: row)
 	}
 
 	convenience init(item: CloudItemMetadata, withParentID parentID: Int64, isPlaceholderItem: Bool = false) {
 		self.init(name: item.name, type: item.itemType, size: item.size, parentID: parentID, lastModifiedDate: item.lastModifiedDate, statusCode: .isUploaded, cloudPath: item.cloudPath, isPlaceholderItem: isPlaceholderItem)
 	}
 
-	init(id: Int64? = nil, name: String, type: CloudItemType, size: Int?, parentID: Int64, lastModifiedDate: Date?, statusCode: ItemStatus, cloudPath: CloudPath, isPlaceholderItem: Bool, isCandidateForCacheCleanup: Bool = false, favoriteRank: Int64? = nil, tagData: Data? = nil) {
+	init(id: Int64? = nil, name: String, type: CloudItemType, size: Int?, parentID: Int64, lastModifiedDate: Date?, statusCode: ItemStatus, cloudPath: CloudPath, isPlaceholderItem: Bool, isCandidateForCacheCleanup: Bool = false, favoriteRank: Int64? = nil, tagData: Data? = nil, lastEnumeratedAt: Date? = nil) {
 		self.id = id
 		self.name = name
 		self.type = type
@@ -65,6 +67,7 @@ public class ItemMetadata: Record, Codable {
 		self.isMaybeOutdated = isCandidateForCacheCleanup
 		self.favoriteRank = favoriteRank
 		self.tagData = tagData
+		self.lastEnumeratedAt = lastEnumeratedAt
 		super.init()
 	}
 
@@ -85,10 +88,11 @@ public class ItemMetadata: Record, Codable {
 		container[Columns.isMaybeOutdated] = isMaybeOutdated
 		container[Columns.favoriteRank] = favoriteRank
 		container[Columns.tagData] = tagData
+		container[Columns.lastEnumeratedAt] = lastEnumeratedAt
 	}
 
 	enum Columns: String, ColumnExpression {
-		case id, name, type, size, parentID, lastModifiedDate, statusCode, cloudPath, isPlaceholderItem, isMaybeOutdated, favoriteRank, tagData
+		case id, name, type, size, parentID, lastModifiedDate, statusCode, cloudPath, isPlaceholderItem, isMaybeOutdated, favoriteRank, tagData, lastEnumeratedAt
 	}
 
 	static func filterWorkingSet() -> QueryInterfaceRequest<ItemMetadata> {
@@ -99,6 +103,6 @@ public class ItemMetadata: Record, Codable {
 extension ItemStatus: DatabaseValueConvertible {}
 extension ItemMetadata: Equatable {
 	public static func == (lhs: ItemMetadata, rhs: ItemMetadata) -> Bool {
-		lhs.id == rhs.id && lhs.name == rhs.name && lhs.type == rhs.type && lhs.size == rhs.size && lhs.parentID == rhs.parentID && lhs.lastModifiedDate == rhs.lastModifiedDate && lhs.statusCode == rhs.statusCode && lhs.cloudPath == rhs.cloudPath && lhs.isPlaceholderItem == rhs.isPlaceholderItem && lhs.isMaybeOutdated == rhs.isMaybeOutdated && lhs.favoriteRank == rhs.favoriteRank && lhs.tagData == rhs.tagData
+		lhs.id == rhs.id && lhs.name == rhs.name && lhs.type == rhs.type && lhs.size == rhs.size && lhs.parentID == rhs.parentID && lhs.lastModifiedDate == rhs.lastModifiedDate && lhs.statusCode == rhs.statusCode && lhs.cloudPath == rhs.cloudPath && lhs.isPlaceholderItem == rhs.isPlaceholderItem && lhs.isMaybeOutdated == rhs.isMaybeOutdated && lhs.favoriteRank == rhs.favoriteRank && lhs.tagData == rhs.tagData && lhs.lastEnumeratedAt == rhs.lastEnumeratedAt
 	}
 }

CryptomatorFileProvider/DB/ItemMetadataDBManager.swift

@@ -39,6 +39,7 @@ protocol ItemMetadataManager {
 	func getAllCachedMetadataInsideWorkingSet() throws -> [ItemMetadata]
 	func setFavoriteRank(to favoriteRank: Int64?, forItemWithID id: Int64) throws
 	func setTagData(to tagData: Data?, forItemWithID id: Int64) throws
+	func setLastEnumeratedAt(_ date: Date, forItemWithID id: Int64) throws
 }
 
 class ItemMetadataDBManager: ItemMetadataManager {
@@ -166,6 +167,14 @@ class ItemMetadataDBManager: ItemMetadataManager {
 		}
 	}
 
+	func setLastEnumeratedAt(_ date: Date, forItemWithID id: Int64) throws {
+		try database.write { db in
+			let cachedMetadata = try getCachedMetadata(for: id, database: db)
+			cachedMetadata?.lastEnumeratedAt = date
+			try cachedMetadata?.update(db)
+		}
+	}
+
 	private func getCachedMetadata(for id: Int64, database: Database) throws -> ItemMetadata? {
 		return try ItemMetadata.fetchOne(database, key: id)
 	}
@@ -176,6 +185,7 @@ class ItemMetadataDBManager: ItemMetadataManager {
 			metadata.statusCode = cachedMetadata.statusCode
 			metadata.tagData = cachedMetadata.tagData
 			metadata.favoriteRank = cachedMetadata.favoriteRank
+			metadata.lastEnumeratedAt = cachedMetadata.lastEnumeratedAt
 			try metadata.update(database)
 		} else {
 			try metadata.insert(database)

CryptomatorFileProvider/DB/UploadTaskDBManager.swift

@@ -7,6 +7,7 @@
 //
 
 import CryptomatorCloudAccessCore
+import FileProvider
 import Foundation
 import GRDB
 
@@ -19,6 +20,7 @@ protocol UploadTaskManager {
 	func removeTaskRecord(for id: Int64) throws
 	func getTask(for uploadTask: UploadTaskRecord, onURLSessionTaskCreation: URLSessionTaskCreationClosure?) throws -> UploadTask
 	func getActiveUploadTaskRecords() throws -> [UploadTaskRecord]
+	func getRetryableUploadTaskRecords() throws -> [UploadTaskRecord]
 }
 
 extension UploadTaskManager {
@@ -99,12 +101,9 @@ class UploadTaskDBManager: UploadTaskManager {
 
 	func getCorrespondingTaskRecords(ids: [Int64]) throws -> [UploadTaskRecord?] {
 		return try database.read { db in
-			var tasks = [UploadTaskRecord?]()
-			for id in ids {
-				let task = try UploadTaskRecord.fetchOne(db, key: id)
-				tasks.append(task)
-			}
-			return tasks
+			let rows = try UploadTaskRecord.filter(keys: ids).fetchAll(db)
+			let dict = Dictionary(uniqueKeysWithValues: rows.map { ($0.correspondingItem, $0) })
+			return ids.map { dict[$0] }
 		}
 	}
 
@@ -130,4 +129,13 @@ class UploadTaskDBManager: UploadTaskManager {
 				.fetchAll(db)
 		}
 	}
+
+	func getRetryableUploadTaskRecords() throws -> [UploadTaskRecord] {
+		return try database.read { db in
+			return try UploadTaskRecord
+				.filter(UploadTaskRecord.Columns.uploadErrorDomain == NSFileProviderErrorDomain &&
+					UploadTaskRecord.Columns.uploadErrorCode == NSFileProviderError.serverUnreachable.rawValue)
+				.fetchAll(db)
+		}
+	}
 }

CryptomatorFileProvider/DB/WorkingSetObserver.swift

@@ -77,12 +77,6 @@ class WorkingSetObserver: WorkingSetObserving {
 	}
 
 	func createFileProviderItems(from metadataList: [ItemMetadata]) throws -> [FileProviderItem] {
-		let uploadTasks = try uploadTaskManager.getTaskRecords(for: metadataList)
-		return try metadataList.enumerated().map { index, metadata -> FileProviderItem in
-			let localCachedFileInfo = try cachedFileManager.getLocalCachedFileInfo(for: metadata)
-			let newestVersionLocallyCached = localCachedFileInfo?.isCurrentVersion(lastModifiedDateInCloud: metadata.lastModifiedDate) ?? false
-			let localURL = localCachedFileInfo?.localURL
-			return FileProviderItem(metadata: metadata, domainIdentifier: domainIdentifier, newestVersionLocallyCached: newestVersionLocallyCached, localURL: localURL, error: uploadTasks[index]?.failedWithError)
-		}
+		return try FileProviderItem.items(from: metadataList, domainIdentifier: domainIdentifier, uploadTaskManager: uploadTaskManager, cachedFileManager: cachedFileManager)
 	}
 }