Support downloading large (2GB+) files (#1922)

src/@types/global.d.ts

@@ -127,3 +127,25 @@ interface Array<T> {
 interface ReadonlyArray<T> {
   filter<S extends T>(predicate: BooleanConstructor, thisArg?: any): Exclude<S, Falsy>[];
 }
+
+// Missing type definitions for OPFS (Origin Private File System) API
+// https://github.com/WICG/file-system-access/blob/main/AccessHandle.md#accesshandle-idl
+interface FileSystemFileHandle extends FileSystemHandle {
+  readonly kind: 'file';
+  getFile(): Promise<File>;
+  createSyncAccessHandle(): Promise<FileSystemSyncAccessHandle>;
+}
+
+interface FileSystemSyncAccessHandle {
+  read: (buffer: BufferSource, options: FilesystemReadWriteOptions) => number;
+  write: (buffer: BufferSource, options: FilesystemReadWriteOptions) => number;
+
+  truncate: (size: number) => Promise<undefined>;
+  getSize: () => Promise<number>;
+  flush: () => Promise<undefined> ;
+  close: () => Promise<undefined>;
+}
+
+type FilesystemReadWriteOptions = {
+  at: number;
+};

src/components/main/DownloadManager.tsx

@@ -6,6 +6,7 @@ import type { Thread } from '../../global/types';
 import type { ApiMessage } from '../../api/types';
 import { ApiMediaFormat } from '../../api/types';
 
+import { IS_OPFS_SUPPORTED } from '../../util/environment';
 import * as mediaLoader from '../../util/mediaLoader';
 import download from '../../util/download';
 import {
@@ -24,7 +25,7 @@ type StateProps = {
 
 const GLOBAL_UPDATE_DEBOUNCE = 1000;
 
-const MAX_BLOB_SIZE = 0x7FFFFFFF - 1;
+const MAX_BLOB_SAFE_SIZE = 2000 * 1024 * 1024;
 
 const processedMessages = new Set<ApiMessage>();
 const downloadedMessages = new Set<ApiMessage>();
@@ -80,10 +81,9 @@ const DownloadManager: FC<StateProps> = ({
         document, video, audio,
       } = message.content;
       const mediaSize = (document || video || audio)?.size || 0;
-      if (mediaSize > MAX_BLOB_SIZE) {
+      if (mediaSize > MAX_BLOB_SAFE_SIZE && !IS_OPFS_SUPPORTED) {
         showNotification({
-          // eslint-disable-next-line max-len
-          message: 'Downloading files bigger than 2GB currently unsupported due to browser limitations. We are working on fixing this issue as soon as possible.',
+          message: 'Downloading files bigger than 2GB is currently not supported in your browser.',
         });
         handleMessageDownloaded(message);
         return;

src/lib/gramjs/client/downloadFile.ts

@@ -35,6 +35,7 @@ const MIN_CHUNK_SIZE = 4096;
 const DEFAULT_CHUNK_SIZE = 64; // kb
 const ONE_MB = 1024 * 1024;
 const DISCONNECT_SLEEP = 1000;
+const MAX_BUFFER_SAFE_SIZE = 2000 * 1024 * 1024;
 
 // when the sender requests hangs for 60 second we will reimport
 const SENDER_TIMEOUT = 60 * 1000;
@@ -69,6 +70,58 @@ class Foreman {
     }
 }
 
+class FileView {
+    private type: 'memory' | 'opfs';
+
+    private size?: number;
+
+    private buffer?: Buffer;
+
+    private largeFile?: FileSystemFileHandle;
+
+    private largeFileAccessHandle?: FileSystemSyncAccessHandle;
+
+    constructor(size?: number) {
+        this.size = size;
+        this.type = (size && size > MAX_BUFFER_SAFE_SIZE) ? 'opfs' : 'memory';
+    }
+
+    async init() {
+        if (this.type === 'opfs') {
+            if (!FileSystemFileHandle?.prototype.createSyncAccessHandle) {
+                throw new Error('`createSyncAccessHandle` is not available. Cannot download files larger than 2GB.');
+            }
+            const directory = await navigator.storage.getDirectory();
+            const downloadsFolder = await directory.getDirectoryHandle('downloads', { create: true });
+            this.largeFile = await downloadsFolder.getFileHandle(Math.random().toString(), { create: true });
+            this.largeFileAccessHandle = await this.largeFile.createSyncAccessHandle();
+        } else {
+            this.buffer = this.size ? Buffer.alloc(this.size) : Buffer.alloc(0);
+        }
+    }
+
+    write(data: Uint8Array, offset: number) {
+        if (this.type === 'opfs') {
+            this.largeFileAccessHandle!.write(data, { at: offset });
+        } else if (this.size) {
+            for (let i = 0; i < data.length; i++) {
+                if (offset + i >= this.buffer!.length) return;
+                this.buffer!.writeUInt8(data[i], offset + i);
+            }
+        } else {
+            this.buffer = Buffer.concat([this.buffer!, data]);
+        }
+    }
+
+    getData(): Promise<Buffer | File> {
+        if (this.type === 'opfs') {
+            return this.largeFile!.getFile();
+        } else {
+            return Promise.resolve(this.buffer!);
+        }
+    }
+}
+
 export async function downloadFile(
     client: TelegramClient,
     inputLocation: Api.InputFileLocation,
@@ -118,6 +171,7 @@ async function downloadFile2(
     client._log.info(`Downloading file in chunks of ${partSize} bytes`);
 
     const foreman = new Foreman(workers);
+    const fileView = new FileView(end - start + 1);
     const promises: Promise<any>[] = [];
     let offset = start;
     // Used for files with unknown size and for manual cancellations
@@ -131,6 +185,9 @@ async function downloadFile2(
     // Preload sender
     await client.getSender(dcId);
 
+    // Allocate memory
+    await fileView.init();
+
     // eslint-disable-next-line no-constant-condition
     while (true) {
         let limit = partSize;
@@ -188,7 +245,9 @@ async function downloadFile2(
 
                     foreman.releaseWorker();
 
-                    return result.bytes;
+                    fileView.write(result.bytes, offsetMemo - start);
+
+                    return;
                 } catch (err) {
                     if (sender && !sender.isConnected()) {
                         await sleep(DISCONNECT_SLEEP);
@@ -212,8 +271,6 @@ async function downloadFile2(
             break;
         }
     }
-    const results = await Promise.all(promises);
-    const buffers = results.filter(Boolean);
-    const totalLength = end ? (end + 1) - start : undefined;
-    return Buffer.concat(buffers, totalLength);
+    await Promise.all(promises);
+    return fileView.getData();
 }

src/util/environment.ts

@@ -91,6 +91,18 @@ export const IS_WEBM_SUPPORTED = Boolean(TEST_VIDEO.canPlayType('video/webm; cod
 export const DPR = window.devicePixelRatio || 1;
 
 export const MASK_IMAGE_DISABLED = true;
+export const IS_OPFS_SUPPORTED = Boolean(navigator.storage?.getDirectory);
+if (IS_OPFS_SUPPORTED) {
+  // Clear old contents
+  (async () => {
+    try {
+      const directory = await navigator.storage.getDirectory();
+      await directory.removeEntry('downloads', { recursive: true });
+    } catch {
+      // Ignore
+    }
+  })();
+}
 
 export const IS_BACKDROP_BLUR_SUPPORTED = !IS_TEST && (
   CSS.supports('backdrop-filter: blur()') || CSS.supports('-webkit-backdrop-filter: blur()')