Add sitemap support (#18407)

modules/setting/setting.go

@@ -207,6 +207,7 @@ var (
 	// UI settings
 	UI = struct {
 		ExplorePagingNum      int
+		SitemapPagingNum      int
 		IssuePagingNum        int
 		RepoSearchPagingNum   int
 		MembersPagingNum      int
@@ -260,6 +261,7 @@ var (
 		} `ini:"ui.meta"`
 	}{
 		ExplorePagingNum:    20,
+		SitemapPagingNum:    20,
 		IssuePagingNum:      10,
 		RepoSearchPagingNum: 10,
 		MembersPagingNum:    20,

modules/sitemap/sitemap.go

@@ -0,0 +1,69 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package sitemap
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"time"
+)
+
+// sitemapFileLimit contains the maximum size of a sitemap file
+const sitemapFileLimit = 50 * 1024 * 1024
+
+// Url represents a single sitemap entry
+type URL struct {
+	URL     string     `xml:"loc"`
+	LastMod *time.Time `xml:"lastmod,omitempty"`
+}
+
+// SitemapUrl represents a sitemap
+type Sitemap struct {
+	XMLName   xml.Name
+	Namespace string `xml:"xmlns,attr"`
+
+	URLs []URL `xml:"url"`
+}
+
+// NewSitemap creates a sitemap
+func NewSitemap() *Sitemap {
+	return &Sitemap{
+		XMLName:   xml.Name{Local: "urlset"},
+		Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
+	}
+}
+
+// NewSitemap creates a sitemap index.
+func NewSitemapIndex() *Sitemap {
+	return &Sitemap{
+		XMLName:   xml.Name{Local: "sitemapindex"},
+		Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
+	}
+}
+
+// Add adds a URL to the sitemap
+func (s *Sitemap) Add(u URL) {
+	s.URLs = append(s.URLs, u)
+}
+
+// Write writes the sitemap to a response
+func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
+	if len(s.URLs) > 50000 {
+		return 0, fmt.Errorf("The sitemap contains too many URLs: %d", len(s.URLs))
+	}
+	buf := bytes.NewBufferString(xml.Header)
+	if err := xml.NewEncoder(buf).Encode(s); err != nil {
+		return 0, err
+	}
+	if err := buf.WriteByte('\n'); err != nil {
+		return 0, err
+	}
+	if buf.Len() > sitemapFileLimit {
+		return 0, fmt.Errorf("The sitemap is too big: %d", buf.Len())
+	}
+	return buf.WriteTo(w)
+}

modules/sitemap/sitemap_test.go

@@ -0,0 +1,77 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package sitemap
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOk(t *testing.T) {
+	testReal := func(s *Sitemap, name string, urls []URL, expected string) {
+		for _, url := range urls {
+			s.Add(url)
+		}
+		buf := &bytes.Buffer{}
+		_, err := s.WriteTo(buf)
+		assert.NoError(t, nil, err)
+		assert.Equal(t, xml.Header+"<"+name+" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"+expected+"</"+name+">\n", buf.String())
+	}
+	test := func(urls []URL, expected string) {
+		testReal(NewSitemap(), "urlset", urls, expected)
+		testReal(NewSitemapIndex(), "sitemapindex", urls, expected)
+	}
+
+	ts := time.Unix(1651322008, 0).UTC()
+
+	test(
+		[]URL{},
+		"",
+	)
+	test(
+		[]URL{
+			{URL: "https://gitea.io/test1", LastMod: &ts},
+		},
+		"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>",
+	)
+	test(
+		[]URL{
+			{URL: "https://gitea.io/test2", LastMod: nil},
+		},
+		"<url><loc>https://gitea.io/test2</loc></url>",
+	)
+	test(
+		[]URL{
+			{URL: "https://gitea.io/test1", LastMod: &ts},
+			{URL: "https://gitea.io/test2", LastMod: nil},
+		},
+		"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>"+
+			"<url><loc>https://gitea.io/test2</loc></url>",
+	)
+}
+
+func TestTooManyURLs(t *testing.T) {
+	s := NewSitemap()
+	for i := 0; i < 50001; i++ {
+		s.Add(URL{URL: fmt.Sprintf("https://gitea.io/test%d", i)})
+	}
+	buf := &bytes.Buffer{}
+	_, err := s.WriteTo(buf)
+	assert.EqualError(t, err, "The sitemap contains too many URLs: 50001")
+}
+
+func TestSitemapTooBig(t *testing.T) {
+	s := NewSitemap()
+	s.Add(URL{URL: strings.Repeat("b", sitemapFileLimit)})
+	buf := &bytes.Buffer{}
+	_, err := s.WriteTo(buf)
+	assert.EqualError(t, err, "The sitemap is too big: 52428931")
+}