fix: allow removing impersonated headers by passing empty string (#382)

Users can now remove impersonated headers (like `Sec-Fetch-User`) from
requests by passing an empty string as the header value. When an empty
string is provided, the header is filtered out before the request is
sent.

This enables users, e.g., to manually control which `Sec-Fetch-*`
headers should be included in their requests, addressing use cases where
the default impersonated headers don't match the actual request context.

Closes #228

Author: barjin

impit-node/index.d.ts

@@ -313,6 +313,8 @@ cookieJar?: { setCookie: (cookie: string, url: string, cb?: any) => Promise<void
  * Header matching is **case-insensitive** — for example, setting `user-agent` here will override
  * the impersonation `User-Agent` header.
  *
+ * To remove an impersonated header, pass an empty string as the value.
+ *
  * @default `undefined` (no additional headers)
  */
 headers?: Headers | Record<string, string> | [string, string][]
@@ -347,10 +349,12 @@ export interface RequestInit {
    *
    * Can be an object, a Map, or an array of tuples or an instance of the {@link https://developer.mozilla.org/en-US/docs/Web/API/Headers | Headers} class.
    *
-   * These headers have the highest priority — they override both client-level headers
-   * (set via {@link ImpitOptions.headers}) and browser impersonation headers (set via {@link ImpitOptions.browser}).
+   * Note that headers set here will override any default headers set in {@link ImpitOptions.headers}.
+   *
    * Header matching is **case-insensitive** — for example, setting `user-agent` here will override
    * the impersonation `User-Agent` header.
+   *
+   * To remove an impersonated header, pass an empty string as the value.
    */
   headers?: Headers | Record<string, string> | [string, string][]
   /** Request body. Can be a string, Buffer, ArrayBuffer, TypedArray, DataView, Blob, File, URLSearchParams, FormData or ReadableStream. */

impit-node/src/impit_builder.rs

@@ -96,6 +96,13 @@ pub struct ImpitOptions<'a> {
   ///
   /// Can be an object, a Map, or an array of tuples or an instance of the {@link https://developer.mozilla.org/en-US/docs/Web/API/Headers | Headers} class.
   ///
+  /// These headers override any browser impersonation headers (set via the {@link ImpitOptions.browser} option)
+  /// and are in turn overridden by request-specific headers (set via {@link RequestInit.headers}).
+  /// Header matching is **case-insensitive** — for example, setting `user-agent` here will override
+  /// the impersonation `User-Agent` header.
+  ///
+  /// To remove an impersonated header, pass an empty string as the value.
+  ///
   /// @default `undefined` (no additional headers)
   #[napi(ts_type = "Headers | Record<string, string> | [string, string][]")]
   pub headers: Option<Vec<(String, String)>>,

impit-node/src/request.rs

@@ -35,6 +35,11 @@ pub struct RequestInit {
   /// Can be an object, a Map, or an array of tuples or an instance of the {@link https://developer.mozilla.org/en-US/docs/Web/API/Headers | Headers} class.
   ///
   /// Note that headers set here will override any default headers set in {@link ImpitOptions.headers}.
+  ///
+  /// Header matching is **case-insensitive** — for example, setting `user-agent` here will override
+  /// the impersonation `User-Agent` header.
+  ///
+  /// To remove an impersonated header, pass an empty string as the value.
   #[napi(ts_type = "Headers | Record<string, string> | [string, string][]")]
   pub headers: Option<Vec<(String, String)>>,
   #[napi(

impit-node/test/basics.test.ts

@@ -188,15 +188,15 @@ describe.each([
 
         test('impit accepts custom cookie jars', async (t) => {
             const cookieJar = new CookieJar();
-            cookieJar.setCookieSync('preset-cookie=123; Path=/', getHttpBinUrl('/cookies/'));
+            cookieJar.setCookieSync('preset-cookie=123; Path=/', getHttpBinUrl('/cookies'));
 
             const impit = new Impit({
                 cookieJar,
                 browser,
             })
 
             const response1 = await impit.fetch(
-                getHttpBinUrl('/cookies/'),
+                getHttpBinUrl('/cookies'),
             ).then(x => x.json());
 
             t.expect(response1.cookies).toEqual({
@@ -208,7 +208,7 @@ describe.each([
             );
 
             const response2 = await impit.fetch(
-                getHttpBinUrl('/cookies/'),
+                getHttpBinUrl('/cookies'),
             ).then(x => x.json());
 
             t.expect(response2.cookies).toEqual({
@@ -278,6 +278,20 @@ describe.each([
             t.expect(json.headers?.['User-Agent']).toBe('this is impit!');
         })
 
+        test('removing impersonated headers with empty string works', async (t) => {
+            const response = await impit.fetch(
+                getHttpBinUrl('/headers'),
+                {
+                    headers: {
+                        'Sec-Fetch-User': '',
+                    }
+                }
+            );
+            const json = await response.json();
+
+            t.expect(json.headers?.['Sec-Fetch-User']).toBeUndefined();
+        });
+
         test('client-scoped headers work', async (t) => {
             const headers = new Headers();
             headers.set('User-Agent', 'client-scoped user agent');

impit-python/python/impit/impit.pyi

@@ -484,7 +484,7 @@ class Client:
             cookie_jar: Cookie jar to store cookies in.
             cookies: httpx-compatible cookies object.
             headers: Default HTTP headers to include in requests. These override browser impersonation
-                headers and are overridden by per-request headers. Matching is case-insensitive.
+                headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value.
             local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface.
                 Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6).
         """
@@ -504,7 +504,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -524,7 +524,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
 
@@ -545,7 +545,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -565,7 +565,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -585,7 +585,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -605,7 +605,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -625,7 +625,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -645,7 +645,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -668,7 +668,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
             stream: Whether to return a streaming response (default: False)
@@ -702,7 +702,7 @@ class Client:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -825,7 +825,7 @@ class AsyncClient:
             cookie_jar: Cookie jar to store cookies in.
             cookies: httpx-compatible cookies object.
             headers: Default HTTP headers to include in requests. These override browser impersonation
-                headers and are overridden by per-request headers. Matching is case-insensitive.
+                headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value.
             local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface.
                 Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6).
         """
@@ -845,7 +845,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -865,7 +865,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
 
@@ -886,7 +886,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -906,7 +906,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -926,7 +926,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -946,7 +946,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -966,7 +966,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -986,7 +986,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """
@@ -1009,7 +1009,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
             stream: Whether to return a streaming response (default: False)
@@ -1043,7 +1043,7 @@ class AsyncClient:
             url: URL to request
             content: Raw content to send
             data: Form data to send in request body
-            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive)
+            headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value
             timeout: Request timeout in seconds (overrides default timeout)
             force_http3: Force HTTP/3 protocol
         """

impit-python/test/async_client_test.py

@@ -299,6 +299,14 @@ async def test_overwriting_headers_work(self, browser: Browser) -> None:
         assert response.status_code == 200
         assert json.loads(response.text)['headers']['User-Agent'] == 'this is impit!'
 
+    @pytest.mark.asyncio
+    async def test_removing_impersonated_headers_with_empty_string(self, browser: Browser) -> None:
+        impit = AsyncClient(browser=browser)
+
+        response = await impit.get(get_httpbin_url('/headers'), headers={'Sec-Fetch-User': ''})
+        assert response.status_code == 200
+        assert 'Sec-Fetch-User' not in json.loads(response.text)['headers']
+
     @pytest.mark.skip(reason='Flaky under the CI environment')
     @pytest.mark.asyncio
     async def test_http3_works(self, browser: Browser) -> None:

impit-python/test/basic_client_test.py

@@ -276,6 +276,13 @@ def test_overwriting_headers_work(self, browser: Browser) -> None:
         assert response.status_code == 200
         assert response.json()['headers']['User-Agent'] == 'this is impit!'
 
+    def test_removing_impersonated_headers_with_empty_string(self, browser: Browser) -> None:
+        impit = Client(browser=browser)
+
+        response = impit.get(get_httpbin_url('/headers'), headers={'Sec-Fetch-User': ''})
+        assert response.status_code == 200
+        assert 'Sec-Fetch-User' not in response.json()['headers']
+
     @pytest.mark.skip(reason='Flaky under the CI environment')
     def test_http3_works(self, browser: Browser) -> None:
         impit = Client(browser=browser, http3=True)