archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

HA support for various text search

Browse Source
This commit is contained in:
Robin Shen 2022-11-17 13:03:46 +08:00
parent 805cb84d83
commit 6ae6be3b1d
18 changed files with 457 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultEmailAddressManager.java
View File

@ -214,7 +214,8 @@ public class DefaultEmailAddressManager extends BaseEntityManager<EmailAddress>
@Override
public void run() {
cache.put(facade.getId(), facade);
if (cache != null)
cache.put(facade.getId(), facade);
}
});

1
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java
View File

@ -1068,6 +1068,7 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements Iss
Project numberScope = targetProject.getForkRoot();
Long nextNumber = getNextNumber(numberScope);
issue.setOldVersion(issue.getFacade());
issue.setProject(targetProject);
issue.setNumberScope(numberScope);
Long oldNumber = issue.getNumber();

3
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultUserManager.java
View File

@ -336,7 +336,8 @@ public class DefaultUserManager extends BaseEntityManager<User> implements UserM
@Override
public void run() {
cache.put(facade.getId(), facade);
if (cache != null)
cache.put(facade.getId(), facade);
}
});

3
server-core/src/main/java/io/onedev/server/model/Build.java
View File

@ -73,6 +73,7 @@ import io.onedev.server.git.service.RefFacade;
import io.onedev.server.infomanager.CommitInfoManager;
import io.onedev.server.model.support.BuildMetric;
import io.onedev.server.model.support.LabelSupport;
import io.onedev.server.model.support.ProjectBelonging;
import io.onedev.server.model.support.build.JobSecret;
import io.onedev.server.model.support.build.actionauthorization.ActionAuthorization;
import io.onedev.server.model.support.build.actionauthorization.CloseMilestoneAuthorization;
@ -112,7 +113,7 @@ import io.onedev.server.web.util.WicketUtils;
@Index(columnList="o_numberScope_id"), @Index(columnList="o_project_id, " + PROP_COMMIT)},
uniqueConstraints={@UniqueConstraint(columnNames={"o_numberScope_id", PROP_NUMBER})}
)
public class Build extends AbstractEntity
public class Build extends ProjectBelonging
implements Referenceable, AttachmentStorageSupport, LabelSupport<BuildLabel> {
private static final long serialVersionUID = 1L;

4
server-core/src/main/java/io/onedev/server/model/CodeComment.java
View File

@ -34,6 +34,7 @@ import io.onedev.server.infomanager.UserInfoManager;
import io.onedev.server.model.support.CompareContext;
import io.onedev.server.model.support.LastUpdate;
import io.onedev.server.model.support.Mark;
import io.onedev.server.model.support.ProjectBelonging;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.CollectionUtils;
@ -43,7 +44,7 @@ import io.onedev.server.util.CollectionUtils;
@Index(columnList="o_pullRequest_id"),
@Index(columnList=Mark.PROP_COMMIT_HASH), @Index(columnList=Mark.PROP_PATH),
@Index(columnList=PROP_CREATE_DATE), @Index(columnList=LastUpdate.COLUMN_DATE)})
public class CodeComment extends AbstractEntity implements AttachmentStorageSupport {
public class CodeComment extends ProjectBelonging implements AttachmentStorageSupport {
private static final long serialVersionUID = 1L;
@ -137,6 +138,7 @@ public class CodeComment extends AbstractEntity implements AttachmentStorageSupp
private transient Collection<User> participants;
@Override
public Project getProject() {
return project;
}

3
server-core/src/main/java/io/onedev/server/model/Issue.java
View File

@ -64,6 +64,7 @@ import io.onedev.server.infomanager.PullRequestInfoManager;
import io.onedev.server.infomanager.UserInfoManager;
import io.onedev.server.model.support.EntityWatch;
import io.onedev.server.model.support.LastUpdate;
import io.onedev.server.model.support.ProjectBelonging;
import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
@ -92,7 +93,7 @@ import io.onedev.server.web.util.WicketUtils;
//use dynamic update in order not to overwrite other edits while background threads change update date
@DynamicUpdate
@Editable
public class Issue extends AbstractEntity implements Referenceable, AttachmentStorageSupport {
public class Issue extends ProjectBelonging implements Referenceable, AttachmentStorageSupport {
private static final long serialVersionUID = 1L;

3
server-core/src/main/java/io/onedev/server/model/PullRequest.java
View File

@ -58,6 +58,7 @@ import io.onedev.server.model.support.BranchProtection;
import io.onedev.server.model.support.EntityWatch;
import io.onedev.server.model.support.LabelSupport;
import io.onedev.server.model.support.LastUpdate;
import io.onedev.server.model.support.ProjectBelonging;
import io.onedev.server.model.support.pullrequest.MergePreview;
import io.onedev.server.model.support.pullrequest.MergeStrategy;
import io.onedev.server.rest.annotation.Api;
@ -81,7 +82,7 @@ import io.onedev.server.web.util.WicketUtils;
uniqueConstraints={@UniqueConstraint(columnNames={"o_numberScope_id", PROP_NUMBER})})
//use dynamic update in order not to overwrite other edits while background threads change update date
@DynamicUpdate
public class PullRequest extends AbstractEntity
public class PullRequest extends ProjectBelonging
implements Referenceable, AttachmentStorageSupport, LabelSupport<PullRequestLabel> {
private static final long serialVersionUID = 1L;

26
server-core/src/main/java/io/onedev/server/model/support/ProjectBelonging.java Normal file
View File

@ -0,0 +1,26 @@
package io.onedev.server.model.support;
import javax.persistence.MappedSuperclass;
import io.onedev.server.model.AbstractEntity;
import io.onedev.server.model.Project;
import io.onedev.server.util.facade.ProjectBelongingFacade;
@MappedSuperclass
public abstract class ProjectBelonging extends AbstractEntity {
private static final long serialVersionUID = 1L;
public abstract Project getProject();
@Override
public ProjectBelongingFacade getOldVersion() {
return (ProjectBelongingFacade) super.getOldVersion();
}
@Override
public ProjectBelongingFacade getFacade() {
return new ProjectBelongingFacade(getId(), getProject().getId());
}
}

84
server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java
View File

@ -1,7 +1,6 @@
package io.onedev.server.search.entitytext;
import java.io.IOException;
import java.util.ArrayList;
import java.io.ObjectStreamException;
import java.util.List;
import javax.annotation.Nullable;
@ -14,114 +13,87 @@ import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.WildcardQuery;
import io.onedev.server.event.entity.EntityRemoved;
import io.onedev.server.event.pubsub.Listen;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.CodeComment;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.lucene.BooleanQueryBuilder;
import io.onedev.server.util.lucene.LuceneUtils;
@Singleton
public class DefaultCodeCommentTextManager extends EntityTextManager<CodeComment>
public class DefaultCodeCommentTextManager extends ProjectTextManager<CodeComment>
implements CodeCommentTextManager {
private static final String FIELD_PROJECT_ID = "projectId";
private static final String FIELD_PATH = "path";
private static final String FIELD_COMMENT = "comment";
@Inject
public DefaultCodeCommentTextManager(Dao dao, StorageManager storageManager,
BatchWorkManager batchWorkManager, TransactionManager transactionManager) {
super(dao, storageManager, batchWorkManager, transactionManager);
BatchWorkManager batchWorkManager, TransactionManager transactionManager,
ProjectManager projectManager, ClusterManager clusterManager) {
super(dao, storageManager, batchWorkManager, transactionManager, projectManager,
clusterManager);
}
public Object writeReplace() throws ObjectStreamException {
return new ManagedSerializedForm(CodeCommentTextManager.class);
}
@Override
protected int getIndexVersion() {
return 1;
}
@Transactional
@Listen
public void on(EntityRemoved event) {
super.on(event);
if (event.getEntity() instanceof Project) {
Long projectId = event.getEntity().getId();
transactionManager.runAfterCommit(new Runnable() {
@Override
public void run() {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(LongPoint.newExactQuery(FIELD_PROJECT_ID, projectId));
}
});
}
});
}
}
@Override
protected void addFields(Document document, CodeComment entity) {
document.add(new LongPoint(FIELD_PROJECT_ID, entity.getProject().getId()));
document.add(new StringField(FIELD_PATH, entity.getMark().getPath(), Store.NO));
document.add(new TextField(FIELD_COMMENT, entity.getContent(), Store.NO));
}
@Nullable
private Query buildQuery(Project project, String queryString) {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder queryBuilder = new BooleanQueryBuilder();
queryBuilder.add(LongPoint.newExactQuery(FIELD_PROJECT_ID, project.getId()), Occur.MUST);
BooleanQuery.Builder contentQueryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder contentQueryBuilder = new BooleanQueryBuilder();
queryString = LuceneUtils.escape(queryString);
Query pathQuery = new BoostQuery(new WildcardQuery(new Term(FIELD_PATH, "*" + queryString + "*")), 0.75f);
try (Analyzer analyzer = newAnalyzer()) {
contentQueryBuilder.add(pathQuery, Occur.SHOULD);
StandardQueryParser parser = new StandardQueryParser(analyzer);
contentQueryBuilder.add(new BoostQuery(parser.parse(queryString, FIELD_COMMENT), 0.5f), Occur.SHOULD);
} catch (Exception e) {
contentQueryBuilder.add(pathQuery, Occur.SHOULD);
contentQueryBuilder.add(new BoostQuery(getTermQuery(FIELD_COMMENT, queryString), 0.5f), Occur.SHOULD);
contentQueryBuilder.add(
new BoostQuery(parser.parse(queryString, FIELD_COMMENT), 0.5f),
Occur.SHOULD);
} catch (QueryNodeException e) {
throw new RuntimeException(e);
}
contentQueryBuilder.setMinimumNumberShouldMatch(1);
queryBuilder.add(contentQueryBuilder.build(), Occur.MUST);
return queryBuilder.build();
}
@Override
public List<CodeComment> query(@Nullable Project project, String queryString,
int firstResult, int maxResults) {
Query query = buildQuery(project, queryString);
if (query != null)
return search(query, firstResult, maxResults);
else
return new ArrayList<>();
return search(buildQuery(project, queryString), firstResult, maxResults);
}
@Override
public long count(@Nullable Project project, String queryString) {
Query query = buildQuery(project, queryString);
if (query != null)
return count(query);
else
return 0;
return count(buildQuery(project, queryString));
}
}

77
server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java
View File

@ -1,6 +1,6 @@
package io.onedev.server.search.entitytext;
import java.io.IOException;
import java.io.ObjectStreamException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -17,22 +17,21 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Query;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.entitymanager.IssueFieldManager;
import io.onedev.server.entitymanager.IssueLinkManager;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.event.entity.EntityRemoved;
import io.onedev.server.event.pubsub.Listen;
import io.onedev.server.model.Issue;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.security.permission.AccessProject;
@ -40,12 +39,12 @@ import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.ProjectScope;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.criteria.Criteria;
import io.onedev.server.util.lucene.BooleanQueryBuilder;
import io.onedev.server.util.lucene.LuceneUtils;
@Singleton
public class DefaultIssueTextManager extends EntityTextManager<Issue> implements IssueTextManager {
public class DefaultIssueTextManager extends ProjectTextManager<Issue> implements IssueTextManager {
private static final String FIELD_PROJECT_ID = "projectId";
private static final String FIELD_NUMBER = "number";
private static final String FIELD_TITLE = "title";
@ -54,8 +53,6 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
private static final String FIELD_DESCRIPTION = "description";
private final ProjectManager projectManager;
private final IssueFieldManager fieldManager;
private final IssueLinkManager linkManager;
@ -64,46 +61,24 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
public DefaultIssueTextManager(Dao dao, StorageManager storageManager,
BatchWorkManager batchWorkManager, TransactionManager transactionManager,
ProjectManager projectManager, IssueFieldManager fieldManager,
IssueLinkManager linkManager) {
super(dao, storageManager, batchWorkManager, transactionManager);
this.projectManager = projectManager;
IssueLinkManager linkManager, ClusterManager clusterManager) {
super(dao, storageManager, batchWorkManager, transactionManager,
projectManager, clusterManager);
this.fieldManager = fieldManager;
this.linkManager = linkManager;
}
public Object writeReplace() throws ObjectStreamException {
return new ManagedSerializedForm(IssueTextManager.class);
}
@Override
protected int getIndexVersion() {
return 2;
}
@Transactional
@Listen
public void on(EntityRemoved event) {
super.on(event);
if (event.getEntity() instanceof Project) {
Long projectId = event.getEntity().getId();
transactionManager.runAfterCommit(new Runnable() {
@Override
public void run() {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(LongPoint.newExactQuery(FIELD_PROJECT_ID, projectId));
}
});
}
});
}
}
@Override
protected void addFields(Document document, Issue entity) {
document.add(new LongPoint(FIELD_PROJECT_ID, entity.getProject().getId()));
document.add(new LongPoint(FIELD_NUMBER, entity.getNumber()));
document.add(new LongPoint(FIELD_CONFIDENTIAL, entity.isConfidential()?1:0));
document.add(new TextField(FIELD_TITLE, entity.getTitle(), Store.NO));
@ -113,9 +88,9 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
@Nullable
private Query buildQuery(@Nullable ProjectScope projectScope, String queryString) {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder queryBuilder = new BooleanQueryBuilder();
if (projectScope != null) {
BooleanQuery.Builder projectQueryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder projectQueryBuilder = new BooleanQueryBuilder();
Project project = projectScope.getProject();
if (projectScope.isRecursive())
projectQueryBuilder.add(buildQuery(projectManager.getSubtreeIds(project.getId())), Occur.SHOULD);
@ -132,7 +107,6 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
projectQueryBuilder.add(getNonConfidentialQuery(ancestor), Occur.SHOULD);
}
}
projectQueryBuilder.setMinimumNumberShouldMatch(1);
queryBuilder.add(projectQueryBuilder.build(), Occur.MUST);
} else if (!SecurityUtils.isAdministrator()) {
Collection<Long> projectIds = projectManager.getPermittedProjects(new AccessProject()).stream()
@ -145,7 +119,7 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
return null;
}
BooleanQuery.Builder contentQueryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder contentQueryBuilder = new BooleanQueryBuilder();
String numberString = queryString;
if (numberString.startsWith("#"))
@ -162,12 +136,10 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
boosts.put(FIELD_DESCRIPTION, 0.5f);
MultiFieldQueryParser parser = new MultiFieldQueryParser(
new String[] {FIELD_TITLE, FIELD_DESCRIPTION}, analyzer, boosts);
contentQueryBuilder.add(parser.parse(queryString), Occur.SHOULD);
} catch (Exception e) {
contentQueryBuilder.add(getTermQuery(FIELD_TITLE, queryString), Occur.SHOULD);
contentQueryBuilder.add(getTermQuery(FIELD_DESCRIPTION, queryString), Occur.SHOULD);
contentQueryBuilder.add(parser.parse(LuceneUtils.escape(queryString)), Occur.SHOULD);
} catch (ParseException e) {
throw new RuntimeException(e);
}
contentQueryBuilder.setMinimumNumberShouldMatch(1);
queryBuilder.add(contentQueryBuilder.build(), Occur.MUST);
return queryBuilder.build();
@ -187,26 +159,25 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
else
projectIdsWithoutConfidentialIssuePermission.add(projectId);
}
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder queryBuilder = new BooleanQueryBuilder();
queryBuilder.add(Criteria.forManyValues(
FIELD_PROJECT_ID, projectIdsWithConfidentialIssuePermission, allIds), Occur.SHOULD);
BooleanQuery.Builder nonConfidentialQueryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder nonConfidentialQueryBuilder = new BooleanQueryBuilder();
nonConfidentialQueryBuilder.add(Criteria.forManyValues(
FIELD_PROJECT_ID, projectIdsWithoutConfidentialIssuePermission, allIds), Occur.MUST);
nonConfidentialQueryBuilder.add(LongPoint.newExactQuery(FIELD_CONFIDENTIAL, 0L), Occur.MUST);
queryBuilder.add(nonConfidentialQueryBuilder.build(), Occur.SHOULD);
queryBuilder.setMinimumNumberShouldMatch(1);
return queryBuilder.build();
}
}
private BooleanQuery getNonConfidentialQuery(Project project) {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
BooleanQueryBuilder builder = new BooleanQueryBuilder();
builder.add(LongPoint.newExactQuery(FIELD_PROJECT_ID, project.getId()), Occur.MUST);
builder.add(LongPoint.newExactQuery(FIELD_CONFIDENTIAL, 0L), Occur.MUST);
return builder.build();
}
@Override
public List<Issue> query(@Nullable ProjectScope projectScope, String queryString,
boolean loadFieldsAndLinks, int firstResult, int maxResults) {

65
server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java
View File

@ -1,6 +1,6 @@
package io.onedev.server.search.entitytext;
import java.io.IOException;
import java.io.ObjectStreamException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -17,43 +17,39 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Query;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.entitymanager.BuildManager;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.entitymanager.PullRequestReviewManager;
import io.onedev.server.event.entity.EntityRemoved;
import io.onedev.server.event.pubsub.Listen;
import io.onedev.server.model.Project;
import io.onedev.server.model.PullRequest;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.security.permission.ReadCode;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.criteria.Criteria;
import io.onedev.server.util.lucene.BooleanQueryBuilder;
import io.onedev.server.util.lucene.LuceneUtils;
@Singleton
public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest>
public class DefaultPullRequestTextManager extends ProjectTextManager<PullRequest>
implements PullRequestTextManager {
private static final String FIELD_PROJECT_ID = "projectId";
private static final String FIELD_NUMBER = "number";
private static final String FIELD_TITLE = "title";
private static final String FIELD_DESCRIPTION = "description";
private final ProjectManager projectManager;
private final PullRequestReviewManager reviewManager;
private final BuildManager buildManager;
@ -62,45 +58,24 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest
public DefaultPullRequestTextManager(Dao dao, StorageManager storageManager,
BatchWorkManager batchWorkManager, TransactionManager transactionManager,
ProjectManager projectManager, PullRequestReviewManager reviewManager,
BuildManager buildManager) {
super(dao, storageManager, batchWorkManager, transactionManager);
this.projectManager = projectManager;
BuildManager buildManager, ClusterManager clusterManager) {
super(dao, storageManager, batchWorkManager, transactionManager, projectManager,
clusterManager);
this.reviewManager = reviewManager;
this.buildManager = buildManager;
}
public Object writeReplace() throws ObjectStreamException {
return new ManagedSerializedForm(PullRequestTextManager.class);
}
@Override
protected int getIndexVersion() {
return 1;
}
@Transactional
@Listen
public void on(EntityRemoved event) {
super.on(event);
if (event.getEntity() instanceof Project) {
Long projectId = event.getEntity().getId();
transactionManager.runAfterCommit(new Runnable() {
@Override
public void run() {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(LongPoint.newExactQuery(FIELD_PROJECT_ID, projectId));
}
});
}
});
}
}
@Override
protected void addFields(Document document, PullRequest entity) {
document.add(new LongPoint(FIELD_PROJECT_ID, entity.getProject().getId()));
document.add(new LongPoint(FIELD_NUMBER, entity.getNumber()));
document.add(new TextField(FIELD_TITLE, entity.getTitle(), Store.NO));
if (entity.getDescription() != null)
@ -109,7 +84,7 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest
@Nullable
private Query buildQuery(@Nullable Project project, String queryString) {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder queryBuilder = new BooleanQueryBuilder();
if (project != null) {
queryBuilder.add(LongPoint.newExactQuery(FIELD_PROJECT_ID, project.getId()), Occur.MUST);
} else if (!SecurityUtils.isAdministrator()) {
@ -124,7 +99,7 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest
return null;
}
}
BooleanQuery.Builder contentQueryBuilder = new BooleanQuery.Builder();
BooleanQueryBuilder contentQueryBuilder = new BooleanQueryBuilder();
String numberString = queryString;
if (numberString.startsWith("#"))
@ -141,12 +116,10 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest
boosts.put(FIELD_DESCRIPTION, 0.5f);
MultiFieldQueryParser parser = new MultiFieldQueryParser(
new String[] {FIELD_TITLE, FIELD_DESCRIPTION}, analyzer, boosts);
contentQueryBuilder.add(parser.parse(queryString), Occur.SHOULD);
} catch (Exception e) {
contentQueryBuilder.add(getTermQuery(FIELD_TITLE, queryString), Occur.SHOULD);
contentQueryBuilder.add(getTermQuery(FIELD_DESCRIPTION, queryString), Occur.SHOULD);
contentQueryBuilder.add(parser.parse(LuceneUtils.escape(queryString)), Occur.SHOULD);
} catch (ParseException e) {
throw new RuntimeException(e);
}
contentQueryBuilder.setMinimumNumberShouldMatch(1);
queryBuilder.add(contentQueryBuilder.build(), Occur.MUST);
return queryBuilder.build();

305
server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java → server-core/src/main/java/io/onedev/server/search/entitytext/ProjectTextManager.java
View File

@ -2,12 +2,16 @@ package io.onedev.server.search.entitytext;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.annotation.Nullable;
@ -28,9 +32,11 @@ import org.apache.lucene.analysis.no.NorwegianAnalyzer;
import org.apache.lucene.analysis.pt.PortugueseAnalyzer;
import org.apache.lucene.analysis.ru.RussianAnalyzer;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.sv.SwedishAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
@ -40,6 +46,8 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SearcherManager;
@ -57,12 +65,18 @@ import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.WordUtils;
import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.cluster.ClusterRunnable;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.event.entity.EntityPersisted;
import io.onedev.server.event.entity.EntityRemoved;
import io.onedev.server.event.pubsub.Listen;
import io.onedev.server.event.system.SystemStarted;
import io.onedev.server.event.system.SystemStopping;
import io.onedev.server.model.AbstractEntity;
import io.onedev.server.model.Project;
import io.onedev.server.model.support.ProjectBelonging;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.persistence.annotation.Transactional;
@ -73,10 +87,11 @@ import io.onedev.server.util.ReflectionUtils;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.concurrent.BatchWorker;
import io.onedev.server.util.concurrent.Prioritized;
public abstract class ProjectTextManager<T extends ProjectBelonging> implements Serializable {
public abstract class EntityTextManager<T extends AbstractEntity> {
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(EntityTextManager.class);
private static final Logger logger = LoggerFactory.getLogger(ProjectTextManager.class);
private static final String FIELD_TYPE = "type";
@ -86,6 +101,8 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
private static final String FIELD_ENTITY_ID = "entityId";
protected static final String FIELD_PROJECT_ID = "projectId";
private static final int INDEXING_PRIORITY = 20;
private static final int BATCH_SIZE = 5000;
@ -112,7 +129,7 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
STOP_WORDS.addAll(WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(
SnowballFilter.class, "english_stop.txt", StandardCharsets.UTF_8)));
STOP_WORDS.addAll(WordlistLoader.getWordSet(IOUtils.getDecodingReader(
EntityTextManager.class, "chinese_stop.txt", StandardCharsets.UTF_8)));
ProjectTextManager.class, "chinese_stop.txt", StandardCharsets.UTF_8)));
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -127,13 +144,18 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
private final BatchWorkManager batchWorkManager;
protected final TransactionManager transactionManager;
protected final ProjectManager projectManager;
private final ClusterManager clusterManager;
private volatile SearcherManager searcherManager;
@SuppressWarnings("unchecked")
public EntityTextManager(Dao dao, StorageManager storageManager, BatchWorkManager batchWorkManager,
TransactionManager transactionManager) {
List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(EntityTextManager.class, getClass());
public ProjectTextManager(Dao dao, StorageManager storageManager, BatchWorkManager batchWorkManager,
TransactionManager transactionManager, ProjectManager projectManager,
ClusterManager clusterManager) {
List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(ProjectTextManager.class, getClass());
if (typeArguments.size() == 1 && AbstractEntity.class.isAssignableFrom(typeArguments.get(0))) {
entityClass = (Class<T>) typeArguments.get(0);
} else {
@ -144,6 +166,8 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
this.storageManager = storageManager;
this.batchWorkManager = batchWorkManager;
this.transactionManager = transactionManager;
this.projectManager = projectManager;
this.clusterManager = clusterManager;
}
protected TermQuery getTermQuery(String name, String value) {
@ -214,16 +238,59 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
@Listen
public void on(EntityPersisted event) {
if (entityClass.isAssignableFrom(event.getEntity().getClass())) {
ProjectBelonging projectBelonging = ((ProjectBelonging)event.getEntity());
Long entityId;
if (!event.isNew())
entityId = event.getEntity().getId();
else
entityId = null;
transactionManager.runAfterCommit(new Runnable() {
Long projectId = projectBelonging.getProject().getId();
UUID projectServerUUID = projectManager.getStorageServerUUID(projectId, true);
UUID oldProjectServerUUID;
if (projectBelonging.getOldVersion() != null) {
oldProjectServerUUID = projectManager.getStorageServerUUID(
projectBelonging.getOldVersion().getProjectId(), true);
} else {
oldProjectServerUUID = null;
}
transactionManager.runAfterCommit(new ClusterRunnable() {
private static final long serialVersionUID = 1L;
@Override
public void run() {
batchWorkManager.submit(getBatchWorker(), new IndexWork(INDEXING_PRIORITY, entityId));
if (oldProjectServerUUID != null && !oldProjectServerUUID.equals(projectServerUUID)) {
clusterManager.submitToServer(oldProjectServerUUID, new ClusterTask<Void>() {
private static final long serialVersionUID = 1L;
@Override
public Void call() throws Exception {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(getTerm(FIELD_ENTITY_ID, String.valueOf(entityId)));
}
});
return null;
}
});
}
clusterManager.submitToServer(projectServerUUID, new ClusterTask<Void>() {
private static final long serialVersionUID = 1L;
@Override
public Void call() throws Exception {
batchWorkManager.submit(getBatchWorker(), new IndexWork(INDEXING_PRIORITY, entityId));
return null;
}
});
}
});
@ -235,22 +302,65 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
public void on(EntityRemoved event) {
if (entityClass.isAssignableFrom(event.getEntity().getClass())) {
Long entityId = event.getEntity().getId();
transactionManager.runAfterCommit(new Runnable() {
Long projectId = ((ProjectBelonging)event.getEntity()).getProject().getId();
transactionManager.runAfterCommit(new ClusterRunnable() {
private static final long serialVersionUID = 1L;
@Override
public void run() {
doWithWriter(new WriterRunnable() {
projectManager.submitToProjectServer(projectId, new ClusterTask<Void>() {
private static final long serialVersionUID = 1L;
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(getTerm(FIELD_ENTITY_ID, String.valueOf(entityId)));
}
public Void call() throws Exception {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(getTerm(FIELD_ENTITY_ID, String.valueOf(entityId)));
}
});
return null;
}
});
}
});
} else if (event.getEntity() instanceof Project) {
Long projectId = event.getEntity().getId();
transactionManager.runAfterCommit(new ClusterRunnable() {
private static final long serialVersionUID = 1L;
@Override
public void run() {
projectManager.submitToProjectServer(projectId, new ClusterTask<Void>() {
private static final long serialVersionUID = 1L;
@Override
public Void call() throws Exception {
doWithWriter(new WriterRunnable() {
@Override
public void run(IndexWriter writer) throws IOException {
writer.deleteDocuments(LongPoint.newExactQuery(FIELD_PROJECT_ID, projectId));
}
});
return null;
}
});
}
});
}
}
protected void doWithWriter(WriterRunnable runnable) {
@ -299,7 +409,7 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
return null;
}
}
private BatchWorker getBatchWorker() {
return new BatchWorker("index" + entityClass.getSimpleName()) {
@ -308,17 +418,22 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
String entityName = WordUtils.uncamel(entityClass.getSimpleName()).toLowerCase();
logger.debug("Indexing {}s...", entityName);
boolean checkNewEntities = false;
Collection<Long> entityIds = new HashSet<>();
for (Prioritized work : works) {
Long entityId = ((IndexWork) work).getEntityId();
if (entityId != null)
entityIds.add(entityId);
else
checkNewEntities = true;
}
index(entityIds);
// do the work batch by batch to avoid consuming too much memory
while (index());
if (checkNewEntities) {
// do the work batch by batch to avoid consuming too much memory
while (index());
}
logger.debug("Indexed {}s", entityName);
}
@ -357,12 +472,14 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
@Override
public void run(IndexWriter writer) throws IOException {
for (T entity : unprocessedEntities)
index(writer, entity);
if (!unprocessedEntities.isEmpty()) {
T lastEntity = unprocessedEntities.get(unprocessedEntities.size() - 1);
T lastEntity = null;
for (T entity: unprocessedEntities) {
if (clusterManager.getLocalServerUUID().equals(projectManager.getStorageServerUUID(entity.getProject().getId(), false)))
index(writer, entity);
lastEntity = entity;
}
if (lastEntity != null) {
Document document = new Document();
document.add(new StringField(FIELD_TYPE, FIELD_LAST_ENTITY_ID, Store.NO));
document.add(new StoredField(FIELD_LAST_ENTITY_ID, String.valueOf(lastEntity.getId())));
@ -378,57 +495,128 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
}
}
protected long count(Query query) {
private Query parse(String queryString) {
try {
IndexSearcher indexSearcher = searcherManager.acquire();
try {
TotalHitCountCollector collector = new TotalHitCountCollector();
indexSearcher.search(query, collector);
return collector.getTotalHits();
} finally {
searcherManager.release(indexSearcher);
}
} catch (IOException e) {
QueryParser parser = new QueryParser("", new StandardAnalyzer()) {
@Override
protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive,
boolean endInclusive) throws ParseException {
if (field.equals(FIELD_PROJECT_ID))
return LongPoint.newRangeQuery(field, Long.parseLong(part1), Long.parseLong(part2));
else
return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive);
}
};
parser.setAllowLeadingWildcard(true);
return parser.parse(queryString);
} catch (ParseException e) {
throw new RuntimeException(e);
}
}
protected long count(Query query) {
String queryString = query.toString();
return clusterManager.runOnAllServers(new ClusterTask<Long>() {
private static final long serialVersionUID = 1L;
@Override
public Long call() throws Exception {
if (searcherManager != null) {
try {
IndexSearcher indexSearcher = searcherManager.acquire();
try {
TotalHitCountCollector collector = new TotalHitCountCollector();
indexSearcher.search(parse(queryString), collector);
return (long) collector.getTotalHits();
} finally {
searcherManager.release(indexSearcher);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
return 0L;
}
}
}).values().stream().reduce(0L, Long::sum);
}
protected List<T> search(Query query, int firstResult, int maxResults) {
try {
IndexSearcher indexSearcher = searcherManager.acquire();
try {
TopDocs topDocs = indexSearcher.search(query, firstResult + maxResults);
List<Long> entityIds = new ArrayList<>();
int index = firstResult;
while (index < topDocs.scoreDocs.length) {
Document doc = indexSearcher.doc(topDocs.scoreDocs[index].doc);
entityIds.add(Long.valueOf(doc.get(FIELD_ENTITY_ID)));
index++;
}
EntityCriteria<T> criteria = EntityCriteria.of(entityClass);
criteria.add(Restrictions.in(AbstractEntity.PROP_ID, entityIds));
List<T> entities = dao.query(criteria);
Collections.sort(entities, new Comparator<T>() {
String queryString = query.toString();
Map<Long, Float> entityScores = new HashMap<>();
for (var entry: clusterManager.runOnAllServers(new ClusterTask<Map<Long, Float>>() {
@Override
public int compare(T o1, T o2) {
return entityIds.indexOf(o1.getId()) - entityIds.indexOf(o2.getId());
private static final long serialVersionUID = 1L;
@Override
public Map<Long, Float> call() throws Exception {
if (searcherManager != null) {
try {
IndexSearcher indexSearcher = searcherManager.acquire();
try {
Map<Long, Float> entityScores = new HashMap<>();
TopDocs topDocs = indexSearcher.search(parse(queryString), firstResult + maxResults);
for (var scoreDoc: topDocs.scoreDocs) {
Document doc = indexSearcher.doc(scoreDoc.doc);
entityScores.put(Long.valueOf(doc.get(FIELD_ENTITY_ID)), scoreDoc.score);
}
return entityScores;
} finally {
searcherManager.release(indexSearcher);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
return entities;
} finally {
searcherManager.release(indexSearcher);
} else {
return new HashMap<>();
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}).entrySet()) {
entityScores.putAll(entry.getValue());
}
List<Long> entityIds = new ArrayList<>(entityScores.keySet());
Collections.sort(entityIds, new Comparator<>() {
@Override
public int compare(Object o1, Object o2) {
if (entityScores.get(o1) < entityScores.get(o2))
return 1;
else
return -1;
}
});
if (firstResult < entityIds.size()) {
EntityCriteria<T> criteria = EntityCriteria.of(entityClass);
criteria.add(Restrictions.in(
AbstractEntity.PROP_ID,
entityIds.subList(firstResult, Math.min(firstResult + maxResults, entityIds.size()))));
List<T> entities = dao.query(criteria);
Collections.sort(entities, new Comparator<T>() {
@Override
public int compare(T o1, T o2) {
return entityIds.indexOf(o1.getId()) - entityIds.indexOf(o2.getId());
}
});
return entities;
} else {
return new ArrayList<>();
}
}
private void index(IndexWriter writer, T entity) throws IOException {
Document document = new Document();
document.add(new StringField(FIELD_ENTITY_ID, String.valueOf(entity.getId()), Store.YES));
document.add(new LongPoint(FIELD_PROJECT_ID, entity.getProject().getId()));
addFields(document, entity);
writer.updateDocument(getTerm(FIELD_ENTITY_ID, String.valueOf(entity.getId())), document);
}
@ -458,4 +646,5 @@ public abstract class EntityTextManager<T extends AbstractEntity> {
abstract void run(IndexWriter writer) throws IOException;
}
}

1
server-core/src/main/java/io/onedev/server/util/criteria/Criteria.java
View File

@ -76,7 +76,6 @@ public abstract class Criteria<T> implements Serializable {
});
queryBuilder.setMinimumNumberShouldMatch(1);
return queryBuilder.build();
}

11
server-core/src/main/java/io/onedev/server/util/facade/BuildFacade.java
View File

@ -1,26 +1,19 @@
package io.onedev.server.util.facade;
public class BuildFacade extends EntityFacade {
public class BuildFacade extends ProjectBelongingFacade {
private static final long serialVersionUID = 1L;
private final Long projectId;
private final Long number;
private final String commitHash;
public BuildFacade(Long id, Long projectId, Long number, String commitHash) {
super(id);
this.projectId = projectId;
super(id, projectId);
this.number = number;
this.commitHash = commitHash;
}
public Long getProjectId() {
return projectId;
}
public Long getNumber() {
return number;
}

18
server-core/src/main/java/io/onedev/server/util/facade/ProjectBelongingFacade.java Normal file
View File

@ -0,0 +1,18 @@
package io.onedev.server.util.facade;
public class ProjectBelongingFacade extends EntityFacade {
private static final long serialVersionUID = 1L;
private final Long projectId;
public ProjectBelongingFacade(Long id, Long projectId) {
super(id);
this.projectId = projectId;
}
public Long getProjectId() {
return projectId;
}
}

39
server-core/src/main/java/io/onedev/server/util/lucene/BooleanQueryBuilder.java Normal file
View File

@ -0,0 +1,39 @@
package io.onedev.server.util.lucene;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanQuery.Builder;
import org.apache.lucene.search.Query;
public class BooleanQueryBuilder extends BooleanQuery.Builder {
@Override
public Builder add(Query query, Occur occur) {
if (query != null && !LuceneUtils.isEmpty(query))
super.add(query, occur);
return this;
}
@Override
public Builder add(BooleanClause clause) {
if (clause.getQuery() != null && !LuceneUtils.isEmpty(clause.getQuery()))
super.add(clause);
return this;
}
@Override
public BooleanQuery build() {
BooleanQuery query = super.build();
if (!LuceneUtils.isEmpty(query))
return query;
else
return null;
}
@Override
public Builder setMinimumNumberShouldMatch(int min) {
throw new UnsupportedOperationException();
}
}

38
server-core/src/main/java/io/onedev/server/util/lucene/LuceneUtils.java Normal file
View File

@ -0,0 +1,38 @@
package io.onedev.server.util.lucene;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.bouncycastle.util.Arrays;
public class LuceneUtils {
private static final char[] SPECIAL_CHARS = new char[] {'+', '-', '&', '|', '!', '(', ')',
'{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/'};
public static String escape(String queryString) {
StringBuilder builder = new StringBuilder();
for (char ch: queryString.toCharArray()) {
if (Arrays.contains(SPECIAL_CHARS, ch))
builder.append('\\');
builder.append(ch);
}
return builder.toString();
}
public static boolean isEmpty(Query query) {
AtomicBoolean hasFields = new AtomicBoolean(false);
query.visit(new QueryVisitor() {
@Override
public boolean acceptField(String field) {
hasFields.set(true);
return super.acceptField(field);
}
});
return !hasFields.get();
}
}

2
server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestDetailPage.html
View File

@ -2,7 +2,9 @@
<div class="pull-request-detail card m-2 m-sm-5">
<div wicket:id="requestHeader" class="card-header align-items-center justify-content-start flex-nowrap">
<div class="d-flex align-items-center flex-grow-1">
<wicket:enclosure child="title">
<div class="card-title mr-3"><span wicket:id="title"></span></div>
</wicket:enclosure>
<a wicket:id="edit" title="Edit title" class="btn btn-xs btn-icon btn-light btn-hover-primary edit mr-3"><wicket:svg href="edit" class="icon"></wicket:svg></a>
<form wicket:id="editForm" class="form flex-grow-1 d-flex align-items-center">
<div class="clearable-wrapper mr-3 flex-grow-1">