Add sitemap support (#18407)

routers/web/explore/repo.go

@@ -11,7 +11,9 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/sitemap"
 )
 
 const (
@@ -30,11 +32,21 @@ type RepoSearchOptions struct {
 
 // RenderRepoSearch render repositories search page
 func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
-	page := ctx.FormInt("page")
+	// Sitemap index for sitemap paths
+	page := int(ctx.ParamsInt64("idx"))
+	isSitemap := ctx.Params("idx") != ""
+	if page <= 1 {
+		page = ctx.FormInt("page")
+	}
+
 	if page <= 0 {
 		page = 1
 	}
 
+	if isSitemap {
+		opts.PageSize = setting.UI.SitemapPagingNum
+	}
+
 	var (
 		repos   []*repo_model.Repository
 		count   int64
@@ -100,6 +112,18 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
 		ctx.ServerError("SearchRepository", err)
 		return
 	}
+	if isSitemap {
+		m := sitemap.NewSitemap()
+		for _, item := range repos {
+			m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
+		}
+		ctx.Resp.Header().Set("Content-Type", "text/xml")
+		if _, err := m.WriteTo(ctx.Resp); err != nil {
+			log.Error("Failed writing sitemap: %v", err)
+		}
+		return
+	}
+
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Total"] = count
 	ctx.Data["Repos"] = repos

routers/web/explore/user.go

@@ -12,7 +12,9 @@ import (
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/sitemap"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -33,11 +35,20 @@ func isKeywordValid(keyword string) bool {
 
 // RenderUserSearch render user search page
 func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions, tplName base.TplName) {
-	opts.Page = ctx.FormInt("page")
+	// Sitemap index for sitemap paths
+	opts.Page = int(ctx.ParamsInt64("idx"))
+	isSitemap := ctx.Params("idx") != ""
+	if opts.Page <= 1 {
+		opts.Page = ctx.FormInt("page")
+	}
 	if opts.Page <= 1 {
 		opts.Page = 1
 	}
 
+	if isSitemap {
+		opts.PageSize = setting.UI.SitemapPagingNum
+	}
+
 	var (
 		users   []*user_model.User
 		count   int64
@@ -73,6 +84,18 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions,
 			return
 		}
 	}
+	if isSitemap {
+		m := sitemap.NewSitemap()
+		for _, item := range users {
+			m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
+		}
+		ctx.Resp.Header().Set("Content-Type", "text/xml")
+		if _, err := m.WriteTo(ctx.Resp); err != nil {
+			log.Error("Failed writing sitemap: %v", err)
+		}
+		return
+	}
+
 	ctx.Data["Keyword"] = opts.Keyword
 	ctx.Data["Total"] = count
 	ctx.Data["Users"] = users

routers/web/home.go

@@ -7,11 +7,18 @@ package web
 
 import (
 	"net/http"
+	"strconv"
 
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/sitemap"
+	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web/middleware"
 	"code.gitea.io/gitea/routers/web/auth"
 	"code.gitea.io/gitea/routers/web/user"
@@ -59,6 +66,52 @@ func Home(ctx *context.Context) {
 	ctx.HTML(http.StatusOK, tplHome)
 }
 
+// HomeSitemap renders the main sitemap
+func HomeSitemap(ctx *context.Context) {
+	m := sitemap.NewSitemapIndex()
+	if !setting.Service.Explore.DisableUsersPage {
+		_, cnt, err := user_model.SearchUsers(&user_model.SearchUserOptions{
+			Type:        user_model.UserTypeIndividual,
+			ListOptions: db.ListOptions{PageSize: 1},
+			IsActive:    util.OptionalBoolTrue,
+			Visible:     []structs.VisibleType{structs.VisibleTypePublic},
+		})
+		if err != nil {
+			ctx.ServerError("SearchUsers", err)
+			return
+		}
+		count := int(cnt)
+		idx := 1
+		for i := 0; i < count; i += setting.UI.SitemapPagingNum {
+			m.Add(sitemap.URL{URL: setting.AppURL + "explore/users/sitemap-" + strconv.Itoa(idx) + ".xml"})
+			idx++
+		}
+	}
+
+	_, cnt, err := repo_model.SearchRepository(&repo_model.SearchRepoOptions{
+		ListOptions: db.ListOptions{
+			PageSize: 1,
+		},
+		Actor:     ctx.Doer,
+		AllPublic: true,
+	})
+	if err != nil {
+		ctx.ServerError("SearchRepository", err)
+		return
+	}
+	count := int(cnt)
+	idx := 1
+	for i := 0; i < count; i += setting.UI.SitemapPagingNum {
+		m.Add(sitemap.URL{URL: setting.AppURL + "explore/repos/sitemap-" + strconv.Itoa(idx) + ".xml"})
+		idx++
+	}
+
+	ctx.Resp.Header().Set("Content-Type", "text/xml")
+	if _, err := m.WriteTo(ctx.Resp); err != nil {
+		log.Error("Failed writing sitemap: %v", err)
+	}
+}
+
 // NotFound render 404 page
 func NotFound(ctx *context.Context) {
 	ctx.Data["Title"] = "Page Not Found"

routers/web/web.go

@@ -294,6 +294,7 @@ func RegisterRoutes(m *web.Route) {
 	// Routers.
 	// for health check
 	m.Get("/", Home)
+	m.Get("/sitemap.xml", ignExploreSignIn, HomeSitemap)
 	m.Group("/.well-known", func() {
 		m.Get("/openid-configuration", auth.OIDCWellKnown)
 		m.Group("", func() {
@@ -310,7 +311,9 @@ func RegisterRoutes(m *web.Route) {
 			ctx.Redirect(setting.AppSubURL + "/explore/repos")
 		})
 		m.Get("/repos", explore.Repos)
+		m.Get("/repos/sitemap-{idx}.xml", explore.Repos)
 		m.Get("/users", explore.Users)
+		m.Get("/users/sitemap-{idx}.xml", explore.Users)
 		m.Get("/organizations", explore.Organizations)
 		m.Get("/code", explore.Code)
 		m.Get("/topics/search", explore.TopicSearch)