finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor and enhance issue indexer to support both searching, filtering and paging (#26012)

Browse source 
Fix #24662.

Replace #24822 and #25708 (although it has been merged)


## Background

In the past, Gitea supported issue searching with a keyword and
conditions in a less efficient way. It worked by searching for issues
with the keyword and obtaining limited IDs (as it is heavy to get all)
on the indexer (bleve/elasticsearch/meilisearch), and then querying with
conditions on the database to find a subset of the found IDs. This is
why the results could be incomplete.

To solve this issue, we need to store all fields that could be used as
conditions in the indexer and support both keyword and additional
conditions when searching with the indexer.

## Major changes

- Redefine `IndexerData` to include all fields that could be used as
filter conditions.
- Refactor `Search(ctx context.Context, kw string, repoIDs []int64,
limit, start int, state string)` to `Search(ctx context.Context, options
*SearchOptions)`, so it supports more conditions now.
- Change the data type stored in `issueIndexerQueue`. Use
`IndexerMetadata` instead of `IndexerData` in case the data has been
updated while it is in the queue. This also reduces the storage size of
the queue.
- Enhance searching with Bleve/Elasticsearch/Meilisearch, make them
fully support `SearchOptions`. Also, update the data versions.
- Keep most logic of database indexer, but remove
`issues.SearchIssueIDsByKeyword` in `models` to avoid confusion where is
the entry point to search issues.
- Start a Meilisearch instance to test it in unit tests.
- Add unit tests with almost full coverage to test
Bleve/Elasticsearch/Meilisearch indexer.

---------

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
Jason Song 2023-07-31 14:28:53 +08:00 • committed by GitHub
parent aba9096999
commit 1e76a824bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 2965 additions and 861 deletions
Download patch file Download diff file Expand all files Collapse all files

330
routers/api/v1/repo/issue.go
View file

@ -132,74 +132,73 @@ func SearchIssues(ctx *context.APIContext) {
isClosed = util.OptionalBoolFalse
}
// find repos user can access (for issue search)
opts := &repo_model.SearchRepoOptions{
Private: false,
AllPublic: true,
TopicOnly: false,
Collaborate: util.OptionalBoolNone,
// This needs to be a column that is not nil in fixtures or
// MySQL will return different results when sorting by null in some cases
OrderBy: db.SearchOrderByAlphabetically,
Actor: ctx.Doer,
}
if ctx.IsSigned {
opts.Private = true
opts.AllLimited = true
}
if ctx.FormString("owner") != "" {
owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner"))
if err != nil {
if user_model.IsErrUserNotExist(err) {
ctx.Error(http.StatusBadRequest, "Owner not found", err)
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err)
var (
repoIDs []int64
allPublic bool
)
{
// find repos user can access (for issue search)
opts := &repo_model.SearchRepoOptions{
Private: false,
AllPublic: true,
TopicOnly: false,
Collaborate: util.OptionalBoolNone,
// This needs to be a column that is not nil in fixtures or
// MySQL will return different results when sorting by null in some cases
OrderBy: db.SearchOrderByAlphabetically,
Actor: ctx.Doer,
}
if ctx.IsSigned {
opts.Private = true
opts.AllLimited = true
}
if ctx.FormString("owner") != "" {
owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner"))
if err != nil {
if user_model.IsErrUserNotExist(err) {
ctx.Error(http.StatusBadRequest, "Owner not found", err)
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err)
}
return
}
return
opts.OwnerID = owner.ID
opts.AllLimited = false
opts.AllPublic = false
opts.Collaborate = util.OptionalBoolFalse
}
opts.OwnerID = owner.ID
opts.AllLimited = false
opts.AllPublic = false
opts.Collaborate = util.OptionalBoolFalse
}
if ctx.FormString("team") != "" {
if ctx.FormString("owner") == "" {
ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team")
return
}
team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team"))
if err != nil {
if organization.IsErrTeamNotExist(err) {
ctx.Error(http.StatusBadRequest, "Team not found", err)
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err)
if ctx.FormString("team") != "" {
if ctx.FormString("owner") == "" {
ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team")
return
}
team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team"))
if err != nil {
if organization.IsErrTeamNotExist(err) {
ctx.Error(http.StatusBadRequest, "Team not found", err)
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err)
}
return
}
opts.TeamID = team.ID
}
if opts.AllPublic {
allPublic = true
opts.AllPublic = false // set it false to avoid returning too many repos, we could filter by indexer
}
repoIDs, _, err = repo_model.SearchRepositoryIDs(opts)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err)
return
}
opts.TeamID = team.ID
}
repoCond := repo_model.SearchRepositoryCondition(opts)
repoIDs, _, err := repo_model.SearchRepositoryIDs(opts)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err)
return
}
var issues []*issues_model.Issue
var filteredCount int64
keyword := ctx.FormTrim("q")
if strings.IndexByte(keyword, 0) >= 0 {
keyword = ""
}
var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return
}
}
var isPull util.OptionalBool
switch ctx.FormString("type") {
@ -211,16 +210,33 @@ func SearchIssues(ctx *context.APIContext) {
isPull = util.OptionalBoolNone
}
labels := ctx.FormTrim("labels")
var includedLabelNames []string
if len(labels) > 0 {
includedLabelNames = strings.Split(labels, ",")
var includedAnyLabels []int64
{
labels := ctx.FormTrim("labels")
var includedLabelNames []string
if len(labels) > 0 {
includedLabelNames = strings.Split(labels, ",")
}
includedAnyLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err)
return
}
}
milestones := ctx.FormTrim("milestones")
var includedMilestones []string
if len(milestones) > 0 {
includedMilestones = strings.Split(milestones, ",")
var includedMilestones []int64
{
milestones := ctx.FormTrim("milestones")
var includedMilestoneNames []string
if len(milestones) > 0 {
includedMilestoneNames = strings.Split(milestones, ",")
}
includedMilestones, err = issues_model.GetMilestoneIDsByNames(ctx, includedMilestoneNames)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetMilestoneIDsByNames", err)
return
}
}
// this api is also used in UI,
@ -232,64 +248,64 @@ func SearchIssues(ctx *context.APIContext) {
limit = setting.API.MaxResponseItems
}
// Only fetch the issues if we either don't have a keyword or the search returned issues
// This would otherwise return all issues if no issues were found by the search.
if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 {
issuesOpt := &issues_model.IssuesOptions{
ListOptions: db.ListOptions{
Page: ctx.FormInt("page"),
PageSize: limit,
},
RepoCond: repoCond,
IsClosed: isClosed,
IssueIDs: issueIDs,
IncludedLabelNames: includedLabelNames,
IncludeMilestones: includedMilestones,
SortType: "priorityrepo",
PriorityRepoID: ctx.FormInt64("priority_repo_id"),
IsPull: isPull,
UpdatedBeforeUnix: before,
UpdatedAfterUnix: since,
}
searchOpt := &issue_indexer.SearchOptions{
Paginator: &db.ListOptions{
PageSize: limit,
Page: ctx.FormInt("page"),
},
Keyword: keyword,
RepoIDs: repoIDs,
AllPublic: allPublic,
IsPull: isPull,
IsClosed: isClosed,
IncludedAnyLabelIDs: includedAnyLabels,
MilestoneIDs: includedMilestones,
SortBy: issue_indexer.SortByCreatedDesc,
}
ctxUserID := int64(0)
if ctx.IsSigned {
ctxUserID = ctx.Doer.ID
}
if since != 0 {
searchOpt.UpdatedAfterUnix = &since
}
if before != 0 {
searchOpt.UpdatedBeforeUnix = &before
}
// Filter for: Created by User, Assigned to User, Mentioning User, Review of User Requested
if ctx.IsSigned {
ctxUserID := ctx.Doer.ID
if ctx.FormBool("created") {
issuesOpt.PosterID = ctxUserID
searchOpt.PosterID = &ctxUserID
}
if ctx.FormBool("assigned") {
issuesOpt.AssigneeID = ctxUserID
searchOpt.AssigneeID = &ctxUserID
}
if ctx.FormBool("mentioned") {
issuesOpt.MentionedID = ctxUserID
searchOpt.MentionID = &ctxUserID
}
if ctx.FormBool("review_requested") {
issuesOpt.ReviewRequestedID = ctxUserID
searchOpt.ReviewRequestedID = &ctxUserID
}
if ctx.FormBool("reviewed") {
issuesOpt.ReviewedID = ctxUserID
}
if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "Issues", err)
return
}
issuesOpt.ListOptions = db.ListOptions{
Page: -1,
}
if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "CountIssues", err)
return
searchOpt.ReviewedID = &ctxUserID
}
}
ctx.SetLinkHeader(int(filteredCount), limit)
ctx.SetTotalCountHeader(filteredCount)
// FIXME: It's unsupported to sort by priority repo when searching by indexer,
// it's indeed an regression, but I think it is worth to support filtering by indexer first.
_ = ctx.FormInt64("priority_repo_id")
ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssues", err)
return
}
issues, err := issues_model.GetIssuesByIDs(ctx, ids, true)
if err != nil {
ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err)
return
}
ctx.SetLinkHeader(int(total), limit)
ctx.SetTotalCountHeader(total)
ctx.JSON(http.StatusOK, convert.ToAPIIssueList(ctx, issues))
}
@ -384,23 +400,12 @@ func ListIssues(ctx *context.APIContext) {
isClosed = util.OptionalBoolFalse
}
var issues []*issues_model.Issue
var filteredCount int64
keyword := ctx.FormTrim("q")
if strings.IndexByte(keyword, 0) >= 0 {
keyword = ""
}
var issueIDs []int64
var labelIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return
}
}
var labelIDs []int64
if splitted := strings.Split(ctx.FormString("labels"), ","); len(splitted) > 0 {
labelIDs, err = issues_model.GetLabelIDsInRepoByNames(ctx.Repo.Repository.ID, splitted)
if err != nil {
@ -465,40 +470,61 @@ func ListIssues(ctx *context.APIContext) {
return
}
// Only fetch the issues if we either don't have a keyword or the search returned issues
// This would otherwise return all issues if no issues were found by the search.
if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 {
issuesOpt := &issues_model.IssuesOptions{
ListOptions: listOptions,
RepoIDs: []int64{ctx.Repo.Repository.ID},
IsClosed: isClosed,
IssueIDs: issueIDs,
LabelIDs: labelIDs,
MilestoneIDs: mileIDs,
IsPull: isPull,
UpdatedBeforeUnix: before,
UpdatedAfterUnix: since,
PosterID: createdByID,
AssigneeID: assignedByID,
MentionedID: mentionedByID,
}
if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "Issues", err)
return
}
issuesOpt.ListOptions = db.ListOptions{
Page: -1,
}
if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "CountIssues", err)
return
searchOpt := &issue_indexer.SearchOptions{
Paginator: &listOptions,
Keyword: keyword,
RepoIDs: []int64{ctx.Repo.Repository.ID},
IsPull: isPull,
IsClosed: isClosed,
SortBy: issue_indexer.SortByCreatedDesc,
}
if since != 0 {
searchOpt.UpdatedAfterUnix = &since
}
if before != 0 {
searchOpt.UpdatedBeforeUnix = &before
}
if len(labelIDs) == 1 && labelIDs[0] == 0 {
searchOpt.NoLabelOnly = true
} else {
for _, labelID := range labelIDs {
if labelID > 0 {
searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID)
} else {
searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID)
}
}
}
ctx.SetLinkHeader(int(filteredCount), listOptions.PageSize)
ctx.SetTotalCountHeader(filteredCount)
if len(mileIDs) == 1 && mileIDs[0] == db.NoConditionID {
searchOpt.MilestoneIDs = []int64{0}
} else {
searchOpt.MilestoneIDs = mileIDs
}
if createdByID > 0 {
searchOpt.PosterID = &createdByID
}
if assignedByID > 0 {
searchOpt.AssigneeID = &assignedByID
}
if mentionedByID > 0 {
searchOpt.MentionID = &mentionedByID
}
ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssues", err)
return
}
issues, err := issues_model.GetIssuesByIDs(ctx, ids, true)
if err != nil {
ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err)
return
}
ctx.SetLinkHeader(int(total), listOptions.PageSize)
ctx.SetTotalCountHeader(total)
ctx.JSON(http.StatusOK, convert.ToAPIIssueList(ctx, issues))
}

2
routers/api/v1/repo/issue_label.go
View file

@ -309,7 +309,7 @@ func prepareForReplaceOrAdd(ctx *context.APIContext, form api.IssueLabelsOption)
return nil, nil, err
}
labels, err := issues_model.GetLabelsByIDs(form.Labels)
labels, err := issues_model.GetLabelsByIDs(form.Labels, "id", "repo_id", "org_id")
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetLabelsByIDs", err)
return nil, nil, err

438
routers/web/repo/issue.go
View file

@ -150,7 +150,6 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
mentionedID int64
reviewRequestedID int64
reviewedID int64
forceEmpty bool
)
if ctx.IsSigned {
@ -191,31 +190,14 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
keyword = ""
}
var issueIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword, ctx.FormString("state"))
if err != nil {
if issue_indexer.IsAvailable(ctx) {
ctx.ServerError("issueIndexer.Search", err)
return
}
ctx.Data["IssueIndexerUnavailable"] = true
}
if len(issueIDs) == 0 {
forceEmpty = true
}
}
var mileIDs []int64
if milestoneID > 0 || milestoneID == db.NoConditionID { // -1 to get those issues which have no any milestone assigned
mileIDs = []int64{milestoneID}
}
var issueStats *issues_model.IssueStats
if forceEmpty {
issueStats = &issues_model.IssueStats{}
} else {
issueStats, err = issues_model.GetIssueStats(&issues_model.IssuesOptions{
{
statsOpts := &issues_model.IssuesOptions{
RepoIDs: []int64{repo.ID},
LabelIDs: labelIDs,
MilestoneIDs: mileIDs,
@ -226,12 +208,34 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
ReviewRequestedID: reviewRequestedID,
ReviewedID: reviewedID,
IsPull: isPullOption,
IssueIDs: issueIDs,
})
if err != nil {
ctx.ServerError("GetIssueStats", err)
return
IssueIDs: nil,
}
if keyword != "" {
allIssueIDs, err := issueIDsFromSearch(ctx, keyword, statsOpts)
if err != nil {
if issue_indexer.IsAvailable(ctx) {
ctx.ServerError("issueIDsFromSearch", err)
return
}
ctx.Data["IssueIndexerUnavailable"] = true
return
}
statsOpts.IssueIDs = allIssueIDs
}
if keyword != "" && len(statsOpts.IssueIDs) == 0 {
// So it did search with the keyword, but no issue found.
// Just set issueStats to empty.
issueStats = &issues_model.IssueStats{}
} else {
// So it did search with the keyword, and found some issues. It needs to get issueStats of these issues.
// Or the keyword is empty, so it doesn't need issueIDs as filter, just get issueStats with statsOpts.
issueStats, err = issues_model.GetIssueStats(statsOpts)
if err != nil {
ctx.ServerError("GetIssueStats", err)
return
}
}
}
isShowClosed := ctx.FormString("state") == "closed"
@ -253,12 +257,10 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
}
pager := context.NewPagination(total, setting.UI.IssuePagingNum, page, 5)
var issues []*issues_model.Issue
if forceEmpty {
issues = []*issues_model.Issue{}
} else {
issues, err = issues_model.Issues(ctx, &issues_model.IssuesOptions{
ListOptions: db.ListOptions{
var issues issues_model.IssueList
{
ids, err := issueIDsFromSearch(ctx, keyword, &issues_model.IssuesOptions{
Paginator: &db.ListOptions{
Page: pager.Paginater.Current(),
PageSize: setting.UI.IssuePagingNum,
},
@ -274,16 +276,23 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
IsPull: isPullOption,
LabelIDs: labelIDs,
SortType: sortType,
IssueIDs: issueIDs,
})
if err != nil {
ctx.ServerError("Issues", err)
if issue_indexer.IsAvailable(ctx) {
ctx.ServerError("issueIDsFromSearch", err)
return
}
ctx.Data["IssueIndexerUnavailable"] = true
return
}
issues, err = issues_model.GetIssuesByIDs(ctx, ids, true)
if err != nil {
ctx.ServerError("GetIssuesByIDs", err)
return
}
}
issueList := issues_model.IssueList(issues)
approvalCounts, err := issueList.GetApprovalCounts(ctx)
approvalCounts, err := issues.GetApprovalCounts(ctx)
if err != nil {
ctx.ServerError("ApprovalCounts", err)
return
@ -306,6 +315,11 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
return
}
if err := issues.LoadAttributes(ctx); err != nil {
ctx.ServerError("issues.LoadAttributes", err)
return
}
ctx.Data["Issues"] = issues
ctx.Data["CommitLastStatus"] = lastStatus
ctx.Data["CommitStatuses"] = commitStatuses
@ -429,6 +443,14 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
ctx.Data["Page"] = pager
}
func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) {
ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts))
if err != nil {
return nil, fmt.Errorf("SearchIssues: %w", err)
}
return ids, nil
}
// Issues render issues page
func Issues(ctx *context.Context) {
isPullList := ctx.Params(":type") == "pulls"
@ -2419,74 +2441,73 @@ func SearchIssues(ctx *context.Context) {
isClosed = util.OptionalBoolFalse
}
// find repos user can access (for issue search)
opts := &repo_model.SearchRepoOptions{
Private: false,
AllPublic: true,
TopicOnly: false,
Collaborate: util.OptionalBoolNone,
// This needs to be a column that is not nil in fixtures or
// MySQL will return different results when sorting by null in some cases
OrderBy: db.SearchOrderByAlphabetically,
Actor: ctx.Doer,
}
if ctx.IsSigned {
opts.Private = true
opts.AllLimited = true
}
if ctx.FormString("owner") != "" {
owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner"))
if err != nil {
if user_model.IsErrUserNotExist(err) {
ctx.Error(http.StatusBadRequest, "Owner not found", err.Error())
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error())
var (
repoIDs []int64
allPublic bool
)
{
// find repos user can access (for issue search)
opts := &repo_model.SearchRepoOptions{
Private: false,
AllPublic: true,
TopicOnly: false,
Collaborate: util.OptionalBoolNone,
// This needs to be a column that is not nil in fixtures or
// MySQL will return different results when sorting by null in some cases
OrderBy: db.SearchOrderByAlphabetically,
Actor: ctx.Doer,
}
if ctx.IsSigned {
opts.Private = true
opts.AllLimited = true
}
if ctx.FormString("owner") != "" {
owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner"))
if err != nil {
if user_model.IsErrUserNotExist(err) {
ctx.Error(http.StatusBadRequest, "Owner not found", err.Error())
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error())
}
return
}
return
opts.OwnerID = owner.ID
opts.AllLimited = false
opts.AllPublic = false
opts.Collaborate = util.OptionalBoolFalse
}
opts.OwnerID = owner.ID
opts.AllLimited = false
opts.AllPublic = false
opts.Collaborate = util.OptionalBoolFalse
}
if ctx.FormString("team") != "" {
if ctx.FormString("owner") == "" {
ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team")
return
}
team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team"))
if err != nil {
if organization.IsErrTeamNotExist(err) {
ctx.Error(http.StatusBadRequest, "Team not found", err.Error())
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error())
if ctx.FormString("team") != "" {
if ctx.FormString("owner") == "" {
ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team")
return
}
team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team"))
if err != nil {
if organization.IsErrTeamNotExist(err) {
ctx.Error(http.StatusBadRequest, "Team not found", err.Error())
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error())
}
return
}
opts.TeamID = team.ID
}
if opts.AllPublic {
allPublic = true
opts.AllPublic = false // set it false to avoid returning too many repos, we could filter by indexer
}
repoIDs, _, err = repo_model.SearchRepositoryIDs(opts)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err.Error())
return
}
opts.TeamID = team.ID
}
repoCond := repo_model.SearchRepositoryCondition(opts)
repoIDs, _, err := repo_model.SearchRepositoryIDs(opts)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err.Error())
return
}
var issues []*issues_model.Issue
var filteredCount int64
keyword := ctx.FormTrim("q")
if strings.IndexByte(keyword, 0) >= 0 {
keyword = ""
}
var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error())
return
}
}
var isPull util.OptionalBool
switch ctx.FormString("type") {
@ -2498,19 +2519,39 @@ func SearchIssues(ctx *context.Context) {
isPull = util.OptionalBoolNone
}
labels := ctx.FormTrim("labels")
var includedLabelNames []string
if len(labels) > 0 {
includedLabelNames = strings.Split(labels, ",")
var includedAnyLabels []int64
{
labels := ctx.FormTrim("labels")
var includedLabelNames []string
if len(labels) > 0 {
includedLabelNames = strings.Split(labels, ",")
}
includedAnyLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err.Error())
return
}
}
milestones := ctx.FormTrim("milestones")
var includedMilestones []string
if len(milestones) > 0 {
includedMilestones = strings.Split(milestones, ",")
var includedMilestones []int64
{
milestones := ctx.FormTrim("milestones")
var includedMilestoneNames []string
if len(milestones) > 0 {
includedMilestoneNames = strings.Split(milestones, ",")
}
includedMilestones, err = issues_model.GetMilestoneIDsByNames(ctx, includedMilestoneNames)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetMilestoneIDsByNames", err.Error())
return
}
}
projectID := ctx.FormInt64("project")
var projectID *int64
if v := ctx.FormInt64("project"); v > 0 {
projectID = &v
}
// this api is also used in UI,
// so the default limit is set to fit UI needs
@ -2521,64 +2562,64 @@ func SearchIssues(ctx *context.Context) {
limit = setting.API.MaxResponseItems
}
// Only fetch the issues if we either don't have a keyword or the search returned issues
// This would otherwise return all issues if no issues were found by the search.
if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 {
issuesOpt := &issues_model.IssuesOptions{
ListOptions: db.ListOptions{
Page: ctx.FormInt("page"),
PageSize: limit,
},
RepoCond: repoCond,
IsClosed: isClosed,
IssueIDs: issueIDs,
IncludedLabelNames: includedLabelNames,
IncludeMilestones: includedMilestones,
ProjectID: projectID,
SortType: "priorityrepo",
PriorityRepoID: ctx.FormInt64("priority_repo_id"),
IsPull: isPull,
UpdatedBeforeUnix: before,
UpdatedAfterUnix: since,
}
searchOpt := &issue_indexer.SearchOptions{
Paginator: &db.ListOptions{
Page: ctx.FormInt("page"),
PageSize: limit,
},
Keyword: keyword,
RepoIDs: repoIDs,
AllPublic: allPublic,
IsPull: isPull,
IsClosed: isClosed,
IncludedAnyLabelIDs: includedAnyLabels,
MilestoneIDs: includedMilestones,
ProjectID: projectID,
SortBy: issue_indexer.SortByCreatedDesc,
}
ctxUserID := int64(0)
if ctx.IsSigned {
ctxUserID = ctx.Doer.ID
}
if since != 0 {
searchOpt.UpdatedAfterUnix = &since
}
if before != 0 {
searchOpt.UpdatedBeforeUnix = &before
}
// Filter for: Created by User, Assigned to User, Mentioning User, Review of User Requested
if ctx.IsSigned {
ctxUserID := ctx.Doer.ID
if ctx.FormBool("created") {
issuesOpt.PosterID = ctxUserID
searchOpt.PosterID = &ctxUserID
}
if ctx.FormBool("assigned") {
issuesOpt.AssigneeID = ctxUserID
searchOpt.AssigneeID = &ctxUserID
}
if ctx.FormBool("mentioned") {
issuesOpt.MentionedID = ctxUserID
searchOpt.MentionID = &ctxUserID
}
if ctx.FormBool("review_requested") {
issuesOpt.ReviewRequestedID = ctxUserID
searchOpt.ReviewRequestedID = &ctxUserID
}
if ctx.FormBool("reviewed") {
issuesOpt.ReviewedID = ctxUserID
}
if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "Issues", err.Error())
return
}
issuesOpt.ListOptions = db.ListOptions{
Page: -1,
}
if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, "CountIssues", err.Error())
return
searchOpt.ReviewedID = &ctxUserID
}
}
ctx.SetTotalCountHeader(filteredCount)
// FIXME: It's unsupported to sort by priority repo when searching by indexer,
// it's indeed an regression, but I think it is worth to support filtering by indexer first.
_ = ctx.FormInt64("priority_repo_id")
ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error())
return
}
issues, err := issues_model.GetIssuesByIDs(ctx, ids, true)
if err != nil {
ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error())
return
}
ctx.SetTotalCountHeader(total)
ctx.JSON(http.StatusOK, convert.ToIssueList(ctx, issues))
}
@ -2620,23 +2661,12 @@ func ListIssues(ctx *context.Context) {
isClosed = util.OptionalBoolFalse
}
var issues []*issues_model.Issue
var filteredCount int64
keyword := ctx.FormTrim("q")
if strings.IndexByte(keyword, 0) >= 0 {
keyword = ""
}
var issueIDs []int64
var labelIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
}
}
var labelIDs []int64
if splitted := strings.Split(ctx.FormString("labels"), ","); len(splitted) > 0 {
labelIDs, err = issues_model.GetLabelIDsInRepoByNames(ctx.Repo.Repository.ID, splitted)
if err != nil {
@ -2675,11 +2705,9 @@ func ListIssues(ctx *context.Context) {
}
}
projectID := ctx.FormInt64("project")
listOptions := db.ListOptions{
Page: ctx.FormInt("page"),
PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")),
var projectID *int64
if v := ctx.FormInt64("project"); v > 0 {
projectID = &v
}
var isPull util.OptionalBool
@ -2706,40 +2734,64 @@ func ListIssues(ctx *context.Context) {
return
}
// Only fetch the issues if we either don't have a keyword or the search returned issues
// This would otherwise return all issues if no issues were found by the search.
if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 {
issuesOpt := &issues_model.IssuesOptions{
ListOptions: listOptions,
RepoIDs: []int64{ctx.Repo.Repository.ID},
IsClosed: isClosed,
IssueIDs: issueIDs,
LabelIDs: labelIDs,
MilestoneIDs: mileIDs,
ProjectID: projectID,
IsPull: isPull,
UpdatedBeforeUnix: before,
UpdatedAfterUnix: since,
PosterID: createdByID,
AssigneeID: assignedByID,
MentionedID: mentionedByID,
}
if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
}
issuesOpt.ListOptions = db.ListOptions{
Page: -1,
}
if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
searchOpt := &issue_indexer.SearchOptions{
Paginator: &db.ListOptions{
Page: ctx.FormInt("page"),
PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")),
},
Keyword: keyword,
RepoIDs: []int64{ctx.Repo.Repository.ID},
IsPull: isPull,
IsClosed: isClosed,
ProjectBoardID: projectID,
SortBy: issue_indexer.SortByCreatedDesc,
}
if since != 0 {
searchOpt.UpdatedAfterUnix = &since
}
if before != 0 {
searchOpt.UpdatedBeforeUnix = &before
}
if len(labelIDs) == 1 && labelIDs[0] == 0 {
searchOpt.NoLabelOnly = true
} else {
for _, labelID := range labelIDs {
if labelID > 0 {
searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID)
} else {
searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID)
}
}
}
ctx.SetTotalCountHeader(filteredCount)
if len(mileIDs) == 1 && mileIDs[0] == db.NoConditionID {
searchOpt.MilestoneIDs = []int64{0}
} else {
searchOpt.MilestoneIDs = mileIDs
}
if createdByID > 0 {
searchOpt.PosterID = &createdByID
}
if assignedByID > 0 {
searchOpt.AssigneeID = &assignedByID
}
if mentionedByID > 0 {
searchOpt.MentionID = &mentionedByID
}
ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt)
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error())
return
}
issues, err := issues_model.GetIssuesByIDs(ctx, ids, true)
if err != nil {
ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error())
return
}
ctx.SetTotalCountHeader(total)
ctx.JSON(http.StatusOK, convert.ToIssueList(ctx, issues))
}

129
routers/web/user/home.go
View file

@ -22,6 +22,7 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/context"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
"code.gitea.io/gitea/modules/json"
@ -466,21 +467,16 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
keyword := strings.Trim(ctx.FormString("q"), " ")
ctx.Data["Keyword"] = keyword
// Execute keyword search for issues.
// USING NON-FINAL STATE OF opts FOR A QUERY.
issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts)
if err != nil {
ctx.ServerError("issueIDsFromSearch", err)
return
}
// Ensure no issues are returned if a keyword was provided that didn't match any issues.
var forceEmpty bool
if len(issueIDsFromSearch) > 0 {
opts.IssueIDs = issueIDsFromSearch
} else if len(keyword) > 0 {
forceEmpty = true
accessibleRepos := container.Set[int64]{}
{
ids, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser)
if err != nil {
ctx.ServerError("GetRepoIDsForIssuesOptions", err)
return
}
for _, id := range ids {
accessibleRepos.Add(id)
}
}
// Educated guess: Do or don't show closed issues.
@ -490,12 +486,21 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
// Filter repos and count issues in them. Count will be used later.
// USING NON-FINAL STATE OF opts FOR A QUERY.
var issueCountByRepo map[int64]int64
if !forceEmpty {
issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts)
{
issueIDs, err := issueIDsFromSearch(ctx, keyword, opts)
if err != nil {
ctx.ServerError("CountIssuesByRepo", err)
ctx.ServerError("issueIDsFromSearch", err)
return
}
if len(issueIDs) > 0 { // else, no issues found, just leave issueCountByRepo empty
opts.IssueIDs = issueIDs
issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts)
if err != nil {
ctx.ServerError("CountIssuesByRepo", err)
return
}
opts.IssueIDs = nil // reset, the opts will be used later
}
}
// Make sure page number is at least 1. Will be posted to ctx.Data.
@ -503,14 +508,17 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
if page <= 1 {
page = 1
}
opts.Page = page
opts.PageSize = setting.UI.IssuePagingNum
opts.Paginator = &db.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
}
// Get IDs for labels (a filter option for issues/pulls).
// Required for IssuesOptions.
var labelIDs []int64
selectedLabels := ctx.FormString("labels")
if len(selectedLabels) > 0 && selectedLabels != "0" {
var err error
labelIDs, err = base.StringsToInt64s(strings.Split(selectedLabels, ","))
if err != nil {
ctx.ServerError("StringsToInt64s", err)
@ -521,7 +529,16 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
// Parse ctx.FormString("repos") and remember matched repo IDs for later.
// Gets set when clicking filters on the issues overview page.
opts.RepoIDs = getRepoIDs(ctx.FormString("repos"))
repoIDs := getRepoIDs(ctx.FormString("repos"))
if len(repoIDs) == 0 {
repoIDs = accessibleRepos.Values()
} else {
// Remove repo IDs that are not accessible to the user.
repoIDs = util.SliceRemoveAllFunc(repoIDs, func(v int64) bool {
return !accessibleRepos.Contains(v)
})
}
opts.RepoIDs = repoIDs
// ------------------------------
// Get issues as defined by opts.
@ -529,15 +546,18 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
// Slice of Issues that will be displayed on the overview page
// USING FINAL STATE OF opts FOR A QUERY.
var issues []*issues_model.Issue
if !forceEmpty {
issues, err = issues_model.Issues(ctx, opts)
var issues issues_model.IssueList
{
issueIDs, err := issueIDsFromSearch(ctx, keyword, opts)
if err != nil {
ctx.ServerError("Issues", err)
ctx.ServerError("issueIDsFromSearch", err)
return
}
issues, err = issues_model.GetIssuesByIDs(ctx, issueIDs, true)
if err != nil {
ctx.ServerError("GetIssuesByIDs", err)
return
}
} else {
issues = []*issues_model.Issue{}
}
// ----------------------------------
@ -576,12 +596,12 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
// Fill stats to post to ctx.Data.
// -------------------------------
var issueStats *issues_model.IssueStats
if !forceEmpty {
{
statsOpts := issues_model.IssuesOptions{
User: ctx.Doer,
IsPull: util.OptionalBoolOf(isPullList),
IsClosed: util.OptionalBoolOf(isShowClosed),
IssueIDs: issueIDsFromSearch,
IssueIDs: nil,
IsArchived: util.OptionalBoolFalse,
LabelIDs: opts.LabelIDs,
Org: org,
@ -589,13 +609,29 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
RepoCond: opts.RepoCond,
}
issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts)
if err != nil {
ctx.ServerError("GetUserIssueStats Shown", err)
return
if keyword != "" {
statsOpts.RepoIDs = opts.RepoIDs
allIssueIDs, err := issueIDsFromSearch(ctx, keyword, &statsOpts)
if err != nil {
ctx.ServerError("issueIDsFromSearch", err)
return
}
statsOpts.IssueIDs = allIssueIDs
}
if keyword != "" && len(statsOpts.IssueIDs) == 0 {
// So it did search with the keyword, but no issue found.
// Just set issueStats to empty.
issueStats = &issues_model.IssueStats{}
} else {
// So it did search with the keyword, and found some issues. It needs to get issueStats of these issues.
// Or the keyword is empty, so it doesn't need issueIDs as filter, just get issueStats with statsOpts.
issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts)
if err != nil {
ctx.ServerError("GetUserIssueStats", err)
return
}
}
} else {
issueStats = &issues_model.IssueStats{}
}
// Will be posted to ctx.Data.
@ -629,9 +665,13 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
ctx.Data["IssueRefEndNames"], ctx.Data["IssueRefURLs"] = issue_service.GetRefEndNamesAndURLs(issues, ctx.FormString("RepoLink"))
if err := issues.LoadAttributes(ctx); err != nil {
ctx.ServerError("issues.LoadAttributes", err)
return
}
ctx.Data["Issues"] = issues
approvalCounts, err := issues_model.IssueList(issues).GetApprovalCounts(ctx)
approvalCounts, err := issues.GetApprovalCounts(ctx)
if err != nil {
ctx.ServerError("ApprovalCounts", err)
return
@ -716,21 +756,12 @@ func getRepoIDs(reposQuery string) []int64 {
return repoIDs
}
func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) {
if len(keyword) == 0 {
return []int64{}, nil
}
searchRepoIDs, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser)
func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) {
ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts))
if err != nil {
return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err)
return nil, fmt.Errorf("SearchIssues: %w", err)
}
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state"))
if err != nil {
return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err)
}
return issueIDsFromSearch, nil
return ids, nil
}
func loadRepoByIDs(ctxUser *user_model.User, issueCountByRepo map[int64]int64, unitType unit.Type) (map[int64]*repo_model.Repository, error) {

2
routers/web/user/notification.go
View file

@ -263,7 +263,7 @@ func NotificationSubscriptions(ctx *context.Context) {
return
}
issues, err := issues_model.Issues(ctx, &issues_model.IssuesOptions{
ListOptions: db.ListOptions{
Paginator: &db.ListOptions{
PageSize: setting.UI.IssuePagingNum,
Page: page,
},