Merge pull request #303 from Lissy93/feat/ui-result-accuracy

UI result accuracy

Author: lissy93

api/_common/http.js

@@ -61,7 +61,9 @@ const wrapNetworkError = (error) => {
   return error;
 };
 
-const UA = 'web-check/1.0 (https://web-check.xyz)';
+export const UA =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+  'Chrome/120.0.0.0 Safari/537.36 (compatible; web-check/1.0; +https://web-check.xyz)';
 
 const send = async (method, url, body, opts = {}) => {
   const finalUrl = appendParams(url, opts.params);

api/archives.js

@@ -47,32 +47,32 @@ const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => {
 };
 
 const wayBackHandler = async (url) => {
-  const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`;
+  // collapse=timestamp:8 returns one row per archived day, slashing payloads
+  // (Wikipedia: 25MB/373k rows -> 428KB/6k rows) without losing first/last/change counts
+  const cdxUrl =
+    `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(url)}` +
+    `&output=json&fl=timestamp,statuscode,digest,length&collapse=timestamp:8`;
 
   try {
     const { data } = await httpGet(cdxUrl);
 
-    // Check there's data
     if (!data || !Array.isArray(data) || data.length <= 1) {
       return { skipped: 'Site has never before been archived via the Wayback Machine' };
     }
 
-    // Remove the header row
     data.shift();
 
-    // Process and return the results
     const firstScan = convertTimestampToDate(data[0][0]);
     const lastScan = convertTimestampToDate(data[data.length - 1][0]);
-    const totalScans = data.length;
+    const daysArchived = data.length;
     const changeCount = countPageChanges(data);
     return {
       firstScan,
       lastScan,
-      totalScans,
+      daysArchived,
       changeCount,
       averagePageSize: getAveragePageSize(data),
-      scanFrequency: getScanFrequency(firstScan, lastScan, totalScans, changeCount),
-      scans: data,
+      scanFrequency: getScanFrequency(firstScan, lastScan, daysArchived, changeCount),
       scanUrl: url,
     };
   } catch (err) {

api/carbon.js

@@ -1,11 +1,11 @@
 import middleware from './_common/middleware.js';
+import { UA } from './_common/http.js';
 import { createLogger } from './_common/logger.js';
 
 const log = createLogger('carbon');
 
 const TIMEOUT = 8000;
 const MAX_BYTES = 10 * 1024 * 1024;
-const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)';
 
 // Sustainable Web Design model v3 constants, matches websitecarbon.com formula
 const KWH_PER_GB = 0.81;
@@ -33,7 +33,7 @@ const fetchByteCount = async (url) => {
   const r = await fetch(url, {
     signal: AbortSignal.timeout(TIMEOUT),
     redirect: 'follow',
-    headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*;q=0.1' },
+    headers: { 'user-agent': UA, accept: 'text/html,*/*;q=0.1' },
   });
   if (!r.ok) throw new Error(`status ${r.status}`);
   if (!r.body) return 0;

api/dnssec.js

@@ -7,7 +7,6 @@ const queryDns = async (domain, type) => {
   const res = await httpGet('https://dns.google/resolve', {
     params: { name: domain, type },
     headers: { Accept: 'application/dns-json' },
-    timeout: 5000,
   });
   return res.data;
 };

api/hsts.js

@@ -22,18 +22,12 @@ const evaluate = (header) => {
   return verdict('Site is compatible with the HSTS preload list!', true, header);
 };
 
-const REQUEST_TIMEOUT = 5000;
-
 const hstsHandler = async (url) =>
   new Promise((resolve) => {
     const req = https.request(url, (res) => {
       resolve(evaluate(res.headers['strict-transport-security']));
       res.resume();
     });
-    req.setTimeout(REQUEST_TIMEOUT, () => {
-      req.destroy();
-      resolve({ error: 'HSTS check timed out' });
-    });
     req.on('error', (e) => resolve({ error: `HSTS check failed: ${e.message}` }));
     req.end();
   });

api/quality.js

@@ -8,7 +8,7 @@ const qualityHandler = async (url) => {
   const endpoint =
     `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?` +
     `url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY` +
-    `&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile` +
+    `&category=BEST_PRACTICES&category=SEO&strategy=mobile` +
     `&key=${auth.value}`;
 
   let data;

api/rank.js

@@ -10,10 +10,7 @@ const rankHandler = async (url) => {
     ? { auth: { username: TRANCO_USERNAME, password: TRANCO_API_KEY } }
     : {};
   try {
-    const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, {
-      timeout: 5000,
-      ...auth,
-    });
+    const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, auth);
     if (!response.data?.ranks?.length) {
       return {
         skipped: `${domain} isn't ranked in the top 1 million sites yet`,

api/redirects.js

@@ -1,9 +1,9 @@
 import middleware from './_common/middleware.js';
+import { UA } from './_common/http.js';
 import { upstreamError } from './_common/upstream.js';
 
 const MAX_REDIRECTS = 12;
 const TIMEOUT_MS = 10000;
-const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)';
 
 // Walks the redirect chain manually, recording each Location header as got did
 const redirectsHandler = async (url) => {
@@ -14,7 +14,7 @@ const redirectsHandler = async (url) => {
       const response = await fetch(current, {
         redirect: 'manual',
         signal: AbortSignal.timeout(TIMEOUT_MS),
-        headers: { 'user-agent': USER_AGENT },
+        headers: { 'user-agent': UA },
       });
       if (response.status < 300 || response.status >= 400) {
         if (response.status >= 400) {

api/robots-txt.js

@@ -22,7 +22,8 @@ const robotsHandler = async (url) => {
     const parsed = parseRobotsTxt(res.data || '');
     return parsed.robots.length ? parsed : { skipped: 'No robots.txt rules found for this host' };
   } catch (error) {
-    if (error.response?.status === 404) {
+    const status = error.response?.status;
+    if (status >= 400 && status < 500) {
       return { skipped: 'No robots.txt file present on this host' };
     }
     return upstreamError(error, 'robots.txt fetch');

api/shodan.js

@@ -9,9 +9,7 @@ const shodanHandler = async (url) => {
   if (auth.skipped) return auth;
   const { hostname } = parseTarget(url);
   try {
-    const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`, {
-      timeout: 8000,
-    });
+    const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`);
     return res.data;
   } catch (error) {
     return upstreamError(error, 'Shodan lookup');