crawl_redirects.js

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════
//  BigCommerce → Shopify  |  301 Redirect CSV Generator
//
//  Crawls the live BigCommerce store, collects every URL,
//  maps each one to its Shopify equivalent, and outputs a
//  CSV ready to upload at:
//  Shopify Admin → Online Store → Navigation → URL Redirects → Import
//
//  SETUP:
//    npm install axios cheerio
//    node crawl_redirects.js
//
//  OUTPUT:
//    redirects.csv  — upload this directly to Shopify
//    crawl_log.txt  — full log of every URL found
// ═══════════════════════════════════════════════════════════════

const fs      = require('fs');
const axios   = require('axios');
const cheerio = require('cheerio');

// ─── CONFIG ──────────────────────────────────────────────────────────────────
const CONFIG = {
  baseUrl:      'https://www.barenakedbotanicals.com',
  outputCsv:    './redirects.csv',
  crawlLog:     './crawl_log.txt',
  delayMs:      400,    // polite crawl delay — don't hammer the live store
  maxPages:     2000,   // safety cap
  userAgent:    'Mozilla/5.0 (compatible; MigrationCrawler/1.0)',
};

// ─── URL CLASSIFIER ───────────────────────────────────────────────────────────
// Determines what type of page a URL is and maps it to Shopify's structure.
//
// BigCommerce URL patterns:
//   Product:    /product-slug/                      → /products/product-slug
//   Category:   /category-name/                     → /collections/category-name
//   Blog post:  /blog/post-slug/                    → /blogs/news/post-slug
//   Blog index: /blog/                              → /blogs/news
//   Home:       /                                   → /  (skip)
//   Search:     /search*                            → skip
//   Account:    /account*  /login*  /cart*          → skip

// Known category slugs — built from the screenshot you shared.
// Add any others you spot during the crawl.
const KNOWN_CATEGORIES = new Set([
  'new', 'body', 'baby', 'bath-bars', 'body-butter', 'body-scrubs',
  'hand-body-cream', 'hand-soap', 'rollerball', 'sunscreen',
  'deodorant', 'citrus', 'earthy', 'floral', 'sweet', 'unscented',
  'shop-all-natural-deodorants', 'face', 'cleansers', 'exfoliation',
  'eye-treatment', 'moisturizers', 'serums', 'toners', 'combination-skin',
  'all-products', 'gift-sets', 'bundles', 'sale',
]);

// Paths to skip entirely — not worth redirecting
const SKIP_PATTERNS = [
  /^\/(cart|checkout|account|login|register|logout|search|wishlist)/,
  /\.(jpg|jpeg|png|gif|svg|webp|css|js|pdf|ico|xml|txt)(\?|$)/i,
  /\?(.*)/,   // skip query strings — BigCommerce search/filter URLs
  /^\/blog\/\?/,
];

function shouldSkip(pathname) {
  return SKIP_PATTERNS.some(p => p.test(pathname));
}

function classifyUrl(pathname) {
  // Remove trailing slash for analysis
  const clean = pathname.replace(/\/$/, '') || '/';

  if (clean === '' || clean === '/') return { type: 'home' };

  const parts = clean.split('/').filter(Boolean);

  // Blog post: /blog/something
  if (parts[0] === 'blog' && parts.length === 2) {
    return { type: 'blog_post', slug: parts[1] };
  }

  // Blog index
  if (parts[0] === 'blog' && parts.length === 1) {
    return { type: 'blog_index' };
  }

  // Single-segment slug — could be product or category
  if (parts.length === 1) {
    const slug = parts[0];
    if (KNOWN_CATEGORIES.has(slug)) {
      return { type: 'category', slug };
    }
    // Default assumption: product page
    return { type: 'product', slug };
  }

  // Two-segment: could be category/product or category/subcategory
  if (parts.length === 2) {
    if (KNOWN_CATEGORIES.has(parts[0])) {
      return { type: 'category', slug: parts[1] };
    }
    return { type: 'product', slug: parts[1] };
  }

  return { type: 'unknown', slug: clean };
}

function toShopifyUrl(pathname, classification) {
  const { type, slug } = classification;
  switch (type) {
    case 'product':    return `/products/${slug}`;
    case 'category':   return `/collections/${slug}`;
    case 'blog_post':  return `/blogs/news/${slug}`;
    case 'blog_index': return `/blogs/news`;
    default:           return null; // home and unknown — skip
  }
}

// ─── CRAWLER ─────────────────────────────────────────────────────────────────

const visited  = new Set();
const queue    = ['/'];
const redirects = [];   // { from, to, type }

function normalizeUrl(href, currentPath) {
  try {
    const url = new URL(href, CONFIG.baseUrl);
    // Only follow internal links
    if (url.hostname !== new URL(CONFIG.baseUrl).hostname) return null;
    return url.pathname;
  } catch {
    return null;
  }
}

function extractLinks($, currentPath) {
  const links = [];
  $('a[href]').each((_, el) => {
    const href = $(el).attr('href');
    if (!href) return;
    const pathname = normalizeUrl(href, currentPath);
    if (pathname) links.push(pathname);
  });
  return links;
}

async function crawlPage(pathname) {
  const url = CONFIG.baseUrl + pathname;
  try {
    const res = await axios.get(url, {
      timeout: 15000,
      headers: { 'User-Agent': CONFIG.userAgent },
      maxRedirects: 3,
      validateStatus: s => s < 500,
    });

    if (res.status === 404) {
      log(`  404: ${pathname}`);
      return [];
    }

    const $ = cheerio.load(res.data);
    return extractLinks($, pathname);
  } catch (err) {
    log(`  ERROR: ${pathname} — ${err.message}`);
    return [];
  }
}

// ─── LOGGING ─────────────────────────────────────────────────────────────────

const logStream = fs.createWriteStream(CONFIG.crawlLog, { flags: 'w' });
function log(msg) {
  console.log(msg);
  logStream.write(msg + '\n');
}

// ─── MAIN ─────────────────────────────────────────────────────────────────────

async function run() {
  console.log('═══════════════════════════════════════════════════');
  console.log('  BigCommerce Crawler  |  Redirect CSV Generator');
  console.log(`  Target: ${CONFIG.baseUrl}`);
  console.log('═══════════════════════════════════════════════════\n');

  let pageCount = 0;

  while (queue.length > 0 && pageCount < CONFIG.maxPages) {
    const pathname = queue.shift();

    if (visited.has(pathname)) continue;
    if (shouldSkip(pathname))  continue;

    visited.add(pathname);
    pageCount++;

    const classification = classifyUrl(pathname);
    const shopifyUrl     = toShopifyUrl(pathname, classification);

    const fromUrl = pathname.endsWith('/') ? pathname : pathname + '/';

    if (shopifyUrl) {
      redirects.push({ from: fromUrl, to: shopifyUrl, type: classification.type });
      log(`[${pageCount}] ${classification.type.padEnd(10)} ${fromUrl} → ${shopifyUrl}`);
    } else {
      log(`[${pageCount}] skip       ${pathname}`);
    }

    // Crawl the page and enqueue new links
    const links = await crawlPage(pathname);
    for (const link of links) {
      if (!visited.has(link) && !shouldSkip(link)) {
        queue.push(link);
      }
    }

    await new Promise(r => setTimeout(r, CONFIG.delayMs));
  }

  // ─── Write CSV ──────────────────────────────────────────────────────────────
  const csvLines = ['"redirect_from","redirect_to"'];
  for (const r of redirects) {
    csvLines.push(`"${r.from}","${r.to}"`);
  }
  fs.writeFileSync(CONFIG.outputCsv, csvLines.join('\n') + '\n');

  // ─── Summary ────────────────────────────────────────────────────────────────
  const byType = redirects.reduce((acc, r) => {
    acc[r.type] = (acc[r.type] || 0) + 1;
    return acc;
  }, {});

  console.log('\n═══════════════════════════════════════════════════');
  console.log(`  Crawl complete. Pages visited: ${pageCount}`);
  console.log(`  Redirects generated: ${redirects.length}`);
  Object.entries(byType).forEach(([type, count]) => {
    console.log(`    ${type.padEnd(12)}: ${count}`);
  });
  console.log(`\n  Output: ${CONFIG.outputCsv}`);
  console.log(`  Log:    ${CONFIG.crawlLog}`);
  console.log('\n  Upload redirects.csv to:');
  console.log('  Shopify Admin → Online Store → Navigation → URL Redirects → Import');
  console.log('═══════════════════════════════════════════════════\n');

  logStream.end();
}

run().catch(err => {
  console.error('Fatal:', err.message);
  process.exit(1);
});