improve getEncodingFromHTML

taisukef · taisukef · commit 375f3b20d022 · 2024-09-15T07:33:40.000+09:00
diff --git a/fetchOrLoad.js b/fetchOrLoad.js
@@ -19,16 +19,14 @@ export const escapeURL = (url) => {
   return s;
 };
 
-const getCharsetFromHTML = (bin) => {
-  const s = new TextDecoder().decode(bin);
-  const cs = s.match(/charset="(.+)"/)
-  if (!cs) {
-    return "utf-8";
+export const getEncodingFromHTML = (bin) => {
+  for (const name  of ["encoding", "charset"]) {
+    const s = new TextDecoder().decode(bin);
+    const cs = s.match(new RegExp(name + "=\"(.+)\""));
+    if (!cs || cs[1].length > 20) continue;
+    return cs[1];
   }
-  if (cs[1].length > 20) {
-    return "utf-8";
-  }
-  return cs[1];
+  return "utf-8";
 };
 
 const fetchText = async (url) => {
@@ -38,7 +36,7 @@ const fetchText = async (url) => {
     },
   };
   const bin = await fetchBin(url, opt);
-  const cset = getCharsetFromHTML(bin);
+  const cset = getEncodingFromHTML(bin);
   //console.log(cset);
   //const text = SJIS.decodeAuto(bin);
   //const text = new TextDecoder("euc-jp").decode(bin);
diff --git a/test/fetchOrLoad.test.js b/test/fetchOrLoad.test.js
@@ -1,5 +1,5 @@
 import * as t from "https://deno.land/std/testing/asserts.ts";
-import { escapeURL, fetchOrLoad } from "../fetchOrLoad.js";
+import { escapeURL, fetchOrLoad, getEncodingFromHTML } from "../fetchOrLoad.js";
 
 const make = (n, len = 200) => {
   const ss = [];
@@ -30,3 +30,10 @@ Deno.test("as browser", async () => {
   const html = await fetchOrLoad(url);
   t.assert(html.indexOf("403 Forbidden") !== -1);
 });
+Deno.test("getEncodingFromHTML", () => {
+  const e = s => new TextEncoder().encode(s);
+  t.assert(getEncodingFromHTML(e(`<html encoding="UTF-8">`)), "UTF-8");
+  t.assert(getEncodingFromHTML(e(`<html charset="UTF-8">`)), "UTF-8");
+  t.assert(getEncodingFromHTML(e(`<html encoding="UTF-a" charset="UTF-b">`)), "UTF-a");
+  t.assert(getEncodingFromHTML(e(`<html charset="UTF-b" encoding="UTF-a">`)), "UTF-a");
+});
