optimise diff zip

Author: sunnylqm

bun.lock

@@ -63,7 +63,7 @@
     "fs-extra": "8",
     "global-dirs": "^4.0.0",
     "gradle-to-js": "^2.0.1",
-    "i18next": "^24.2.3",
+    "i18next": "^26.0.4",
     "isomorphic-git": "^1.37.5",
     "isomorphic-unzip": "^1.1.5",
     "node-fetch": "^2.6.1",
@@ -75,7 +75,7 @@
     "registry-auth-token": "^5.1.1",
     "semver": "^7.7.4",
     "tcp-ping": "^0.1.1",
-    "tty-table": "4.2",
+    "tty-table": "5.0",
     "yauzl": "^3.3.0",
     "yazl": "3.3.1"
   },

package.json

@@ -203,10 +203,11 @@ async function diffFromPPK(
   }
 
   //console.log({copies, deletes});
+  const diffManifest = Buffer.from(JSON.stringify({ copies, deletes }));
   zipfile.addBuffer(
-    Buffer.from(JSON.stringify({ copies, deletes })),
+    diffManifest,
     '__diff.json',
-    zipOptionsForManifestEntry(),
+    zipOptionsForManifestEntry(diffManifest.length),
   );
   zipfile.end();
   await writePromise;
@@ -317,10 +318,11 @@ async function diffFromPackage(
     }
   });
 
+  const diffManifest = Buffer.from(JSON.stringify({ copies }));
   zipfile.addBuffer(
-    Buffer.from(JSON.stringify({ copies })),
+    diffManifest,
     '__diff.json',
-    zipOptionsForManifestEntry(),
+    zipOptionsForManifestEntry(diffManifest.length),
   );
   zipfile.end();
   await writePromise;

src/diff.ts

@@ -7,6 +7,7 @@ export type ZipEntryOptions = {
 };
 
 export const ZIP_ENTRY_SNIFF_BYTES = 64;
+export const MANIFEST_COMPRESSION_THRESHOLD_BYTES = 256;
 
 const alreadyCompressedExtensions = new Set([
   '.7z',
@@ -136,7 +137,10 @@ export function zipOptionsForPatchEntry(): ZipEntryOptions {
   return { compress: false };
 }
 
-export function zipOptionsForManifestEntry(): ZipEntryOptions {
+export function zipOptionsForManifestEntry(byteLength = 0): ZipEntryOptions {
+  if (byteLength >= MANIFEST_COMPRESSION_THRESHOLD_BYTES) {
+    return { compressionLevel: 9 };
+  }
   return { compress: false };
 }
 

src/utils/zip-options.ts

@@ -184,6 +184,46 @@ describe('diff commands', () => {
     expect(diffMeta.deletes['old-only.txt']).toBe(1);
   });
 
+  test('diff compresses large manifest entries', async () => {
+    const originPath = path.join(tempRoot, 'origin-large-manifest.ppk');
+    const nextPath = path.join(tempRoot, 'next-large-manifest.ppk');
+    const outputPath = path.join(tempRoot, 'out', 'large-manifest-diff.ppk');
+    const originEntries: Record<string, string | Buffer> = {
+      'index.bundlejs': 'old-bundle',
+    };
+    const nextEntries: Record<string, string | Buffer> = {
+      'index.bundlejs': 'new-bundle',
+    };
+
+    for (let i = 0; i < 24; i++) {
+      const content = `same-content-${i}`;
+      originEntries[`assets/old/path/asset-${i}.txt`] = content;
+      nextEntries[`assets/new/path/renamed-asset-${i}.txt`] = content;
+    }
+
+    await createZip(originPath, originEntries);
+    await createZip(nextPath, nextEntries);
+
+    await diffCommands.diff(
+      createContext([originPath, nextPath], {
+        output: outputPath,
+        customDiff: () => Buffer.from('patch'),
+      }),
+    );
+
+    const result = await readZipContent(outputPath);
+    expect(result.compressionMethods['__diff.json']).toBe(8);
+
+    const diffMeta = JSON.parse(
+      result.files['__diff.json'].toString('utf-8'),
+    ) as {
+      copies: Record<string, string>;
+    };
+    expect(diffMeta.copies['assets/new/path/renamed-asset-0.txt']).toBe(
+      'assets/old/path/asset-0.txt',
+    );
+  });
+
   test('diff throws when origin bundle file is missing', async () => {
     const originPath = path.join(tempRoot, 'origin-no-bundle.ppk');
     const nextPath = path.join(tempRoot, 'next.ppk');

tests/diff.test.ts

@@ -9,6 +9,7 @@ import {
 } from 'bun:test';
 import { translateOptions } from '../src/utils';
 import {
+  MANIFEST_COMPRESSION_THRESHOLD_BYTES,
   zipOptionsForManifestEntry,
   zipOptionsForPatchEntry,
   zipOptionsForPayloadEntry,
@@ -82,8 +83,17 @@ describe('zipOptionsForPatchEntry', () => {
 });
 
 describe('zipOptionsForManifestEntry', () => {
-  test('returns compress false', () => {
+  test('stores small manifests', () => {
     expect(zipOptionsForManifestEntry()).toEqual({ compress: false });
+    expect(
+      zipOptionsForManifestEntry(MANIFEST_COMPRESSION_THRESHOLD_BYTES - 1),
+    ).toEqual({ compress: false });
+  });
+
+  test('compresses manifests at threshold', () => {
+    expect(
+      zipOptionsForManifestEntry(MANIFEST_COMPRESSION_THRESHOLD_BYTES),
+    ).toEqual({ compressionLevel: 9 });
   });
 });