Generate XML Sitemaps Dynamically

Overview

An XML sitemap is the list you hand search engines: which pages exist, when they changed, how much each one matters. A static file works while the site is small; once content changes daily — blogs, stores, user pages — you want the sitemap generated from live data. Below is a working version in three languages: sitemap XML built from a database query or content API, pagination for when you pass the 50,000-URL-per-file limit, and hreflang annotations for multi-language sites.

When to Use

Use this resource when:

• Your site has thousands of pages that change regularly and hand-editing a static sitemap stopped being realistic.
• You run a multi-language site and need xhtml:link annotations in the sitemap for hreflang.
• You want lastmod, changefreq, and priority derived from real content timestamps instead of hardcoded values.
• You’re past — or heading past — 50,000 URLs and need a sitemap index referencing paginated files.

Solution

Python (Flask)

from flask import Flask, Response
from datetime import datetime
import math
import xml.etree.ElementTree as ET

app = Flask(__name__)

def generate_sitemap(urls):
    urlset = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")

    for url_data in urls:
        url_elem = ET.SubElement(urlset, "url")
        ET.SubElement(url_elem, "loc").text = url_data["loc"]
        ET.SubElement(url_elem, "lastmod").text = url_data["lastmod"]
        ET.SubElement(url_elem, "changefreq").text = url_data.get("changefreq", "weekly")
        ET.SubElement(url_elem, "priority").text = str(url_data.get("priority", "0.5"))

    return ET.tostring(urlset, encoding="unicode")

@app.route("/sitemap.xml")
def sitemap():
    # In production, query your database or CMS
    urls = [
        {"loc": "https://example.com/", "lastmod": "2024-06-01", "priority": "1.0"},
        {"loc": "https://example.com/blog/post-1", "lastmod": "2024-06-10", "priority": "0.8"},
        {"loc": "https://example.com/products/item-1", "lastmod": "2024-06-05", "priority": "0.6"},
    ]

    xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
    xml += generate_sitemap(urls)

    return Response(xml, mimetype="application/xml")

# Sitemap index for large sites
@app.route("/sitemap-index.xml")
def sitemap_index():
    sitemap_count = math.ceil(total_url_count / 50000)  # e.g. len(urls) from your DB
    sitemapindex = ET.Element("sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")

    for i in range(1, sitemap_count + 1):
        sitemap_elem = ET.SubElement(sitemapindex, "sitemap")
        ET.SubElement(sitemap_elem, "loc").text = f"https://example.com/sitemap-{i}.xml"
        ET.SubElement(sitemap_elem, "lastmod").text = datetime.now().strftime("%Y-%m-%d")

    xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
    xml += ET.tostring(sitemapindex, encoding="unicode")
    return Response(xml, mimetype="application/xml")

JavaScript (Express)

const express = require("express");
const app = express();

function buildSitemap(urls) {
  const lines = [
    '<?xml version="1.0" encoding="UTF-8"?>',
    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
  ];

  for (const url of urls) {
    lines.push("  <url>");
    lines.push(`    <loc>${escapeXml(url.loc)}</loc>`);
    lines.push(`    <lastmod>${url.lastmod}</lastmod>`);
    lines.push(`    <changefreq>${url.changefreq || "weekly"}</changefreq>`);
    lines.push(`    <priority>${url.priority || "0.5"}</priority>`);
    lines.push("  </url>");
  }

  lines.push("</urlset>");
  return lines.join("\n");
}

function escapeXml(str) {
  return str.replace(/&/g, "&amp;")
            .replace(/</g, "&lt;")
            .replace(/>/g, "&gt;")
            .replace(/"/g, "&quot;")
            .replace(/'/g, "&apos;");
}

app.get("/sitemap.xml", (req, res) => {
  const urls = [
    { loc: "https://example.com/", lastmod: "2024-06-01", priority: "1.0" },
    { loc: "https://example.com/blog/post-1", lastmod: "2024-06-10", priority: "0.8" }
  ];

  res.set("Content-Type", "application/xml");
  res.send(buildSitemap(urls));
});

// Paginated sitemap with hreflang
app.get("/sitemap-products.xml", (req, res) => {
  const products = getProducts(); // From DB
  const lines = [
    '<?xml version="1.0" encoding="UTF-8"?>',
    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"',
    '        xmlns:xhtml="http://www.w3.org/1999/xhtml">'
  ];

  for (const product of products) {
    lines.push("  <url>");
    lines.push(`    <loc>${escapeXml(product.url)}</loc>`);
    lines.push(`    <lastmod>${product.updatedAt}</lastmod>`);

    // hreflang annotations
    for (const lang of ["en", "es", "de"]) {
      lines.push(`    <xhtml:link rel="alternate" hreflang="${lang}"`);
      lines.push(`                href="${escapeXml(product.url)}?lang=${lang}" />`);
    }

    lines.push("  </url>");
  }

  lines.push("</urlset>");
  res.set("Content-Type", "application/xml");
  res.send(lines.join("\n"));
});

Java (Spring Boot)

import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.StringWriter;
import java.time.LocalDate;
import java.util.List;

@RestController
public class SitemapController {

  record UrlEntry(String loc, String lastmod, String changefreq, String priority) {}

  @GetMapping(value = "/sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
  public String sitemap() {
    List<UrlEntry> urls = List.of(
      new UrlEntry("https://example.com/", "2024-06-01", "daily", "1.0"),
      new UrlEntry("https://example.com/blog/post-1", "2024-06-10", "weekly", "0.8"),
      new UrlEntry("https://example.com/products/item-1", "2024-06-05", "weekly", "0.6")
    );

    return buildSitemap(urls);
  }

  private String buildSitemap(List<UrlEntry> urls) {
    try {
      DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
      Document doc = builder.newDocument();

      Element urlset = doc.createElement("urlset");
      urlset.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9");
      doc.appendChild(urlset);

      for (UrlEntry entry : urls) {
        Element url = doc.createElement("url");
        urlset.appendChild(url);

        Element loc = doc.createElement("loc");
        loc.setTextContent(entry.loc());
        url.appendChild(loc);

        Element lastmod = doc.createElement("lastmod");
        lastmod.setTextContent(entry.lastmod());
        url.appendChild(lastmod);

        Element changefreq = doc.createElement("changefreq");
        changefreq.setTextContent(entry.changefreq());
        url.appendChild(changefreq);

        Element priority = doc.createElement("priority");
        priority.setTextContent(entry.priority());
        url.appendChild(priority);
      }

      Transformer transformer = TransformerFactory.newInstance().newTransformer();
      transformer.setOutputProperty(OutputKeys.INDENT, "yes");
      transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");

      StringWriter writer = new StringWriter();
      transformer.transform(new DOMSource(doc), new StreamResult(writer));
      return writer.toString();
    } catch (Exception e) {
      throw new RuntimeException("Failed to generate sitemap", e);
    }
  }
}

Explanation

• XML structure follows the sitemaps.org protocol: a <urlset> root with <url> entries, each carrying <loc>, <lastmod>, <changefreq>, and <priority>. Everything must be UTF-8 and properly escaped — one raw & in a query string breaks the whole file for the crawler.
• Live generation means querying your database, CMS, or file system at request time and rendering fresh XML. On a busy site, don’t rebuild on every hit — cache the output for an hour or two (see Performance Tips).
• Pagination via a sitemap index — one sitemap file tops out at 50,000 URLs or 50MB uncompressed. Past that, serve a sitemapindex file that points at sitemap-1.xml, sitemap-2.xml, and so on.
• Hreflang annotations go inside each <url> as <xhtml:link> elements, and every URL has to list itself plus all of its alternates — canonical included. Get this wrong and your multi-language indexing quietly breaks.

Variants

Approach	Source	Hreflang	Best For
Database query	SQL/ORM	Manual	Content-heavy sites (blogs, CMS)
Static build	File system	Pre-generated	JAMstack/SSG sites (Astro, Next.js)
API-driven	REST/GraphQL	From API metadata	Headless CMS (Contentful, Strapi)
Cached file	Redis/disk	Static after first request	High-traffic sites with stable content

What Works

1. Always escape XML — URLs must XML-escape &, <, >, ", and '. An unescaped ampersand in a query string breaks the sitemap parser.
2. Set accurate lastmod — use the actual content modification date, not the current date. Google ignores inaccurate lastmod values and may stop trusting your sitemap.
3. Paginate before hitting limits — start generating a sitemap index when you approach 40,000 URLs to leave headroom for growth. Each referenced sitemap must be under 50MB.
4. Include canonical URLs only — exclude parameterized variants, session IDs, and 404 pages. Sitemaps should only list canonical, indexable pages.
5. Submit to search engines — register your sitemap in Google Search Console and Bing Webmaster Tools. For very large sites, use the Search Console API to push sitemap updates programmatically.

Common Mistakes

1. Generating sitemaps with http:// URLs when the site uses HTTPS — search engines treat these as separate sites and may ignore the HTTP version.
2. Forgetting to XML-escape URLs with query parameters, causing parser errors in search engine crawlers.
3. Listing noindex pages or redirect chains in the sitemap, which wastes crawl budget and confuses search engines.
4. Using the current date for all lastmod values, making the attribute meaningless and potentially ignored by crawlers.
5. Omitting the sitemap index for sites with 100,000+ URLs, resulting in oversized individual sitemap files that violate the 50,000 URL / 50MB limit.

Production Setup

Robots.txt

Point crawlers at the index, not at the individual sitemap files — one Sitemap: line is enough:

# robots.txt
User-agent: *
Allow: /

Sitemap: https://example.com/sitemap-index.xml

News Sitemap Extension

For Google News, add the news: namespace with publication metadata. Entries can’t be older than 48 hours:

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
        xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
  <url>
    <loc>https://example.com/news/article-1</loc>
    <news:news>
      <news:publication>
        <news:name>Example Times</news:name>
        <news:language>en</news:language>
      </news:publication>
      <news:publication_date>2024-06-15</news:publication_date>
      <news:title>Breaking Story</news:title>
    </news:news>
  </url>
</urlset>

Performance Tips

1. Cache the rendered XML with a short TTL. Rebuilding on every request is wasted work — regenerate every 1-6 hours:

import redis
from functools import wraps
from flask import request

r = redis.Redis(host="localhost", port=6379, db=0)

def cache_sitemap(ttl_seconds=3600):
    def decorator(fn):
        @wraps(fn)
        def wrapper(*args, **kwargs):
            cache_key = f"sitemap:{request.path}"
            cached = r.get(cache_key)
            if cached:
                return Response(cached, mimetype="application/xml")
            xml = fn(*args, **kwargs)
            r.setex(cache_key, ttl_seconds, xml)
            return Response(xml, mimetype="application/xml")
        return wrapper
    return decorator

@app.route("/sitemap.xml")
@cache_sitemap(ttl_seconds=3600)
def sitemap():
    urls = query_urls_from_db()
    return generate_sitemap(urls)

2. Generate sitemaps in a background job for large sites. Write the files to disk on a cron schedule and let the web server serve them as static assets — zero query cost per request:

# background task
def generate_sitemap_files():
    urls = query_all_urls()
    for i, batch in enumerate(split_sitemaps(urls)):
        xml = build_sitemap_xml(batch)
        with open(f"/var/www/sitemap-{i+1}.xml", "w") as f:
            f.write(xml)

3. Serve them gzipped. Sitemap XML compresses 70-90%; either point robots.txt at a .xml.gz file or pre-generate it alongside:

import gzip
from flask import Response

@app.route("/sitemap.xml.gz")
def sitemap_gzipped():
    xml = generate_sitemap(urls)
    compressed = gzip.compress(xml.encode("utf-8"))
    return Response(compressed, mimetype="application/gzip")

# or generate and compress ahead of time
gzip -c sitemap-1.xml > sitemap-1.xml.gz

See Also

• sitemaps.org protocol — the official schema: urlset, url, and sitemapindex elements
• Google Search Central: Sitemaps — limits, formats, and submission
• Google: Localized versions — how xhtml:link alternates in sitemaps interact with hreflang
• Companion: runnable sitemap generators (Flask, Express, Spring Boot)