visual-regression.js

require('dotenv').config();

const backstop = require('backstopjs');
const axios = require('axios');
const https = require('https');

const httpsAgent = new https.Agent({
  // This is required to allow self-signed certificates for local development. In production, you should use a valid certificate.
  rejectUnauthorized: false
});

const referenceDomain = `${process.env.REFERENCE_DOMAIN}`;
const testDomain = `${process.env.TEST_DOMAIN}`;

const backstopViewport = process.env.BACKSTOP_VIEWPORT || 'large';

const backstopAsyncCaptureLimit = Number.parseInt(process.env.BACKSTOP_ASYNC_CAPTURE_LIMIT || '5', 10);
const backstopAsyncCompareLimit = Number.parseInt(process.env.BACKSTOP_ASYNC_COMPARE_LIMIT || '50', 10);
const statusCheckDelayMs = Number.parseInt(process.env.STATUS_CHECK_DELAY_MS || '500', 10);
const statusCheckMaxRetries = Number.parseInt(process.env.STATUS_CHECK_MAX_RETRIES || '3', 10);
const statusCheckRetryBaseDelayMs = Number.parseInt(process.env.STATUS_CHECK_RETRY_BASE_DELAY_MS || '1000', 10);

let { scenarios } = require('./paths');

const { config } = require('./config');

const { allViewports } = require('./viewports');

const setActiveViewports = () => {
  let viewports = [];

  switch (backstopViewport) {
    case 'all':
      for (const viewportKey of Object.keys(allViewports)) {
        viewports.push(allViewports[viewportKey]);
      }
      break;

    default:
      viewports.push(allViewports[backstopViewport]);
  }

  return viewports;
};

console.log(`Using BACKSTOP_VIEWPORT=${backstopViewport} for visual regression tests.`);
const activeViewports = setActiveViewports();

if (!activeViewports || activeViewports.length === 0) {
  console.error(`No valid viewports found for BACKSTOP_VIEWPORT=${backstopViewport}. Please check your .env configuration.`);
  process.exit(1);
}

config.viewports = activeViewports;

config.asyncCaptureLimit = backstopAsyncCaptureLimit;
config.asyncCompareLimit = backstopAsyncCompareLimit;

const referenceConfig = structuredClone(config);
const testConfig = structuredClone(config);

const authUser = process.env.BASIC_AUTH_USER;
const authPass = process.env.BASIC_AUTH_PASSWORD;

const authCreds = authUser && authPass ? `${authUser}:${authPass}@` : '';

scenarios.forEach(scenario => {
  const referenceUrl = `https://${authCreds}${referenceDomain}/${scenario.url}`;
  const testUrl = `https://${authCreds}${testDomain}/${scenario.url}`;

  const referenceScenario = {
    label: scenario.label,
    url: referenceUrl,
  };

  const testScenario = {
    label: scenario.label,
    url: testUrl,
  };

  if (scenario.properties) {
    for (const key of Object.keys(scenario.properties)) {
      referenceScenario[key] = scenario.properties[key];
      testScenario[key] = scenario.properties[key];
    }
  }

  referenceConfig.scenarios.push(referenceScenario);
  testConfig.scenarios.push(testScenario);
});

const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

const getRetryDelayMs = (retryAfterHeader, attempt) => {
  if (retryAfterHeader) {
    const retryAfterSeconds = Number.parseInt(retryAfterHeader, 10);
    if (!Number.isNaN(retryAfterSeconds)) {
      return retryAfterSeconds * 1000;
    }
  }

  return statusCheckRetryBaseDelayMs * (attempt + 1);
};

const headWithRetry = async (url) => {
  let attempt = 0;

  while (attempt <= statusCheckMaxRetries) {
    try {
      return await axios.head(url, { httpsAgent, maxRedirects: 5 });
    } catch (err) {
      const status = err.response?.status;
      if (status !== 429 || attempt === statusCheckMaxRetries) {
        throw err;
      }

      const retryAfter = err.response?.headers?.['retry-after'];
      const waitMs = getRetryDelayMs(retryAfter, attempt);
      console.warn(`Rate limited (429). Retrying in ${waitMs}ms for ${url}`);
      await sleep(waitMs);
      attempt += 1;
    }
  }

  return null;
};

const checkStatusCodes = async () => {
  const allUrls = [
    ...referenceConfig.scenarios.map(s => ({ label: s.label, domain: referenceDomain, url: s.url })),
    ...testConfig.scenarios.map(s => ({ label: s.label, domain: testDomain, url: s.url })),
  ];

  let hasErrors = false;

  for (const { label, domain, url } of allUrls) {
    try {
      const response = await headWithRetry(url);
      if (response.status >= 400) {
        console.error(`HTTP ${response.status} error for "${label}" on ${domain}: ${url}`);
        hasErrors = true;
      } else {
        console.log(`HTTP ${response.status} — "${label}" on ${domain}`);
      }
    } catch (err) {
      const status = err.response?.status;
      if (status) {
        console.error(`HTTP ${status} error for "${label}" on ${domain}: ${url}`);
      } else {
        console.error(`Request failed for "${label}" on ${domain}: ${url} — ${err.message}`);
      }
      hasErrors = true;
    }

    await sleep(statusCheckDelayMs);
  }

  return hasErrors;
};

console.log(`Checking status codes for all URLs on ${referenceDomain} and ${testDomain}...`);

checkStatusCodes().then((hasErrors) => {
  if (hasErrors) {
    console.error('One or more URLs returned an error status code. Aborting visual regression tests.');
    process.exit(1);
  }

  console.log('All URLs returned successful status codes. Proceeding with visual regression tests.');

  console.log(`Running backstop with ${backstopViewport} viewport against ${referenceDomain} to get the reference screenshots.`);

  backstop('reference', { config: referenceConfig })
    .then(() => {
      console.log(`Running backstop against ${testDomain} using ${backstopViewport} viewport to validate that nothing has changed.`);
      backstop('test', { config: testConfig }).then(() => {
        console.log(`Backstop test passed using ${backstopViewport} viewport`);
      }).catch((e) => {
        console.error(`Error in backstop test when comparing ${referenceDomain} with ${testDomain} using ${backstopViewport} viewport`);
        console.log(e);
        process.exit(1);
      });
    }).catch((e) => {
    console.error('Error generating reference screenshots');
    console.log(e);
    process.exit(1);
  });
});