archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

Build is initially set to pending when execute by job executors

Browse Source
This commit is contained in:
Robin Shen 2019-06-19 16:31:33 +08:00
parent 8a0c9d0060
commit ded705cc5c
29 changed files with 297 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
server-core/src/main/java/io/onedev/server/ci/job/DefaultJobManager.java
View File

@ -37,6 +37,7 @@ import io.onedev.commons.launcher.loader.ListenerRegistry;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.LockUtils;
import io.onedev.commons.utils.MatrixRunner;
import io.onedev.commons.utils.schedule.SchedulableTask;
import io.onedev.commons.utils.schedule.TaskScheduler;
import io.onedev.server.OneException;
@ -53,7 +54,7 @@ import io.onedev.server.entitymanager.UserManager;
import io.onedev.server.event.BuildCommitAware;
import io.onedev.server.event.ProjectEvent;
import io.onedev.server.event.build.BuildFinished;
import io.onedev.server.event.build.BuildQueueing;
import io.onedev.server.event.build.BuildPending;
import io.onedev.server.event.build.BuildRunning;
import io.onedev.server.event.build.BuildSubmitted;
import io.onedev.server.event.entity.EntityPersisted;
@ -65,15 +66,15 @@ import io.onedev.server.model.BuildParam;
import io.onedev.server.model.Project;
import io.onedev.server.model.Setting;
import io.onedev.server.model.Setting.Key;
import io.onedev.server.model.User;
import io.onedev.server.model.support.JobExecutionContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.model.support.SourceSnapshot;
import io.onedev.server.model.User;
import io.onedev.server.persistence.SessionManager;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.util.Input;
import io.onedev.server.util.MatrixRunner;
import io.onedev.server.util.inputspec.InputSpec;
import io.onedev.server.util.inputspec.SecretInput;
import io.onedev.server.util.patternset.PatternSet;
@ -150,9 +151,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
});
/*
* Lock to guarantee uniqueness of build (by project, commit, job and parameters)
*/
// Lock to guarantee uniqueness of build (by project, commit, job and parameters)
try {
lock.lockInterruptibly();
return submit(project, commitId, jobName, paramMap, submitter, new LinkedHashSet<>());
@ -241,118 +240,130 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
return null;
}
private void run(Build build) {
private void execute(Build build) {
try {
Job job = build.getJob();
ObjectId commitId = ObjectId.fromString(build.getCommitHash());
JobExecutor executor = getJobExecutor(build.getProject(), commitId, job.getName(), job.getEnvironment());
if (executor != null) {
if (executor.hasCapacity()) {
build.setStatus(Build.Status.RUNNING);
build.setRunningDate(new Date());
buildManager.save(build);
listenerRegistry.post(new BuildRunning(build));
SourceSnapshot snapshot;
if (job.isCloneSource())
snapshot = new SourceSnapshot(build.getProject(), commitId);
else
snapshot = null;
Logger logger = logManager.getLogger(build, job.getLogLevel());
Long buildId = build.getId();
JobExecution execution = new JobExecution(executorService.submit(new Runnable() {
SourceSnapshot snapshot;
if (job.isCloneSource())
snapshot = new SourceSnapshot(build.getProject(), commitId);
else
snapshot = null;
Logger logger = logManager.getLogger(build, job.getLogLevel());
Long buildId = build.getId();
JobExecution execution = new JobExecution(executorService.submit(new Runnable() {
@Override
public void run() {
logger.info("Creating workspace...");
File workspace = FileUtils.createTempDir("workspace");
try {
Map<String, String> envVars = new HashMap<>();
Set<String> includeFiles = new HashSet<>();
Set<String> excludeFiles = new HashSet<>();
sessionManager.run(new Runnable() {
@Override
public void run() {
logger.info("Creating workspace...");
File workspace = FileUtils.createTempDir("workspace");
try {
Map<String, String> envVars = new HashMap<>();
Set<String> includeFiles = new HashSet<>();
Set<String> excludeFiles = new HashSet<>();
sessionManager.run(new Runnable() {
@Override
public void run() {
Build build = buildManager.load(buildId);
logger.info("Populating dependencies...");
for (BuildDependence dependence: build.getDependencies()) {
for (DependencyPopulator populator: dependencyPopulators)
populator.populate(dependence.getDependency(), workspace, logger);
}
envVars.put("ONEDEV_PROJECT", build.getProject().getName());
envVars.put("ONEDEV_COMMIT", commitId.name());
envVars.put("ONEDEV_JOB", job.getName());
envVars.put("ONEDEV_BUILD_NUMBER", String.valueOf(build.getNumber()));
for (Entry<String, Input> entry: build.getParamInputs().entrySet()) {
String paramName = entry.getKey();
if (build.isParamVisible(paramName)) {
String paramType = entry.getValue().getType();
List<String> paramValues = entry.getValue().getValues();
if (paramValues.size() > 1) {
int index = 1;
for (String value: paramValues) {
if (paramType.equals(InputSpec.SECRET))
value = build.getSecretValue(value);
envVars.put(paramName + "_" + index++, value);
}
} else if (paramValues.size() == 1) {
String value = paramValues.iterator().next();
@Override
public void run() {
Build build = buildManager.load(buildId);
logger.info("Populating dependencies...");
for (BuildDependence dependence: build.getDependencies()) {
for (DependencyPopulator populator: dependencyPopulators)
populator.populate(dependence.getDependency(), workspace, logger);
}
envVars.put("ONEDEV_PROJECT", build.getProject().getName());
envVars.put("ONEDEV_COMMIT", commitId.name());
envVars.put("ONEDEV_JOB", job.getName());
envVars.put("ONEDEV_BUILD_NUMBER", String.valueOf(build.getNumber()));
for (Entry<String, Input> entry: build.getParamInputs().entrySet()) {
String paramName = entry.getKey();
if (build.isParamVisible(paramName)) {
String paramType = entry.getValue().getType();
List<String> paramValues = entry.getValue().getValues();
if (paramValues.size() > 1) {
int index = 1;
for (String value: paramValues) {
if (paramType.equals(InputSpec.SECRET))
value = build.getSecretValue(value);
envVars.put(paramName, value);
envVars.put(paramName + "_" + index++, value);
}
} else if (paramValues.size() == 1) {
String value = paramValues.iterator().next();
if (paramType.equals(InputSpec.SECRET))
value = build.getSecretValue(value);
envVars.put(paramName, value);
}
}
for (JobOutcome outcome: job.getOutcomes()) {
PatternSet patternSet = PatternSet.fromString(outcome.getFilePatterns());
includeFiles.addAll(patternSet.getIncludes());
excludeFiles.addAll(patternSet.getExcludes());
}
for (JobOutcome outcome: job.getOutcomes()) {
PatternSet patternSet = PatternSet.fromString(outcome.getFilePatterns());
includeFiles.addAll(patternSet.getIncludes());
excludeFiles.addAll(patternSet.getExcludes());
}
}
});
logger.info("Executing job with executor '" + executor.getName() + "'...");
List<String> commands = Splitter.on("\n").trimResults(CharMatcher.is('\r')).splitToList(job.getCommands());
executor.execute(new JobExecutionContext(job.getEnvironment(), workspace, envVars, commands,
snapshot, job.getCaches(), new PatternSet(includeFiles, excludeFiles), logger) {
@Override
public void notifyJobRunning() {
transactionManager.run(new Runnable() {
@Override
public void run() {
logger.info("Collecting job outcomes...");
Build build = buildManager.load(buildId);
build.setStatus(Build.Status.RUNNING);
build.setRunningDate(new Date());
buildManager.save(build);
listenerRegistry.post(new BuildRunning(build));
}
}
});
logger.info("Executing job with executor '" + executor.getName() + "'...");
});
}
List<String> commands = Splitter.on("\n").trimResults(CharMatcher.is('\r')).splitToList(job.getCommands());
executor.execute(job.getEnvironment(), workspace, envVars, commands, snapshot,
job.getCaches(), new PatternSet(includeFiles, excludeFiles), logger);
sessionManager.run(new Runnable() {
});
sessionManager.run(new Runnable() {
@Override
public void run() {
logger.info("Collecting job outcomes...");
Build build = buildManager.load(buildId);
for (JobOutcome outcome: job.getOutcomes())
outcome.process(build, workspace, logger);
}
});
} catch (Exception e) {
if (ExceptionUtils.find(e, InterruptedException.class) == null)
logger.error("Error running build", e);
throw e;
} finally {
logger.info("Deleting workspace...");
executor.cleanDir(workspace);
FileUtils.deleteDir(workspace);
logger.info("Workspace deleted");
}
@Override
public void run() {
logger.info("Collecting job outcomes...");
Build build = buildManager.load(buildId);
for (JobOutcome outcome: job.getOutcomes())
outcome.process(build, workspace, logger);
}
});
} catch (Exception e) {
if (ExceptionUtils.find(e, InterruptedException.class) == null)
logger.error("Error running build", e);
throw e;
} finally {
logger.info("Deleting workspace...");
executor.cleanDir(workspace);
FileUtils.deleteDir(workspace);
logger.info("Workspace deleted");
}
}), job.getTimeout() * 1000L);
}
JobExecution prevExecution = jobExecutions.put(build.getId(), execution);
if (prevExecution != null)
prevExecution.cancel(null);
}
}), job.getTimeout() * 1000L);
JobExecution prevExecution = jobExecutions.put(build.getId(), execution);
if (prevExecution != null)
prevExecution.cancel(null);
} else {
markBuildError(build, "No applicable job executor");
}
@ -405,7 +416,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
if (build.isFinished()) {
build.setStatus(Build.Status.WAITING);
build.setFinishDate(null);
build.setQueueingDate(null);
build.setPendingDate(null);
build.setRunningDate(null);
build.setSubmitDate(new Date());
build.setSubmitter(submitter);
@ -497,10 +508,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
@Override
public Boolean call() {
for (Build build: buildManager.queryUnfinished()) {
if (build.getStatus() == Build.Status.QUEUEING) {
if (status == Status.STARTED)
run(build);
} else if (build.getStatus() == Build.Status.RUNNING) {
if (build.getStatus() == Build.Status.PENDING || build.getStatus() == Build.Status.RUNNING) {
JobExecution execution = jobExecutions.get(build.getId());
if (execution != null) {
if (execution.isTimedout())
@ -526,9 +534,12 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
if (hasUnsuccessful) {
markBuildError(build, "There are failed dependency jobs");
} else if (!hasUnfinished) {
build.setStatus(Build.Status.QUEUEING);
build.setQueueingDate(new Date());
listenerRegistry.post(new BuildQueueing(build));
build.setStatus(Build.Status.PENDING);
build.setPendingDate(new Date());
listenerRegistry.post(new BuildPending(build));
if (status == Status.STARTED)
execute(build);
}
}
}
@ -536,7 +547,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask
Map.Entry<Long, JobExecution> entry = it.next();
Build build = buildManager.get(entry.getKey());
JobExecution execution = entry.getValue();
if (build == null || build.getStatus() != Build.Status.RUNNING) {
if (build == null || build.getStatus() != Build.Status.PENDING && build.getStatus() != Build.Status.RUNNING) {
it.remove();
execution.cancel(null);
} else if (execution.isDone()) {

2
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultBuildManager.java
View File

@ -161,7 +161,7 @@ public class DefaultBuildManager extends AbstractEntityManager<Build> implements
public Collection<Build> queryUnfinished() {
EntityCriteria<Build> criteria = newCriteria();
criteria.add(Restrictions.or(
Restrictions.eq("status", Status.QUEUEING),
Restrictions.eq("status", Status.PENDING),
Restrictions.eq("status", Status.RUNNING),
Restrictions.eq("status", Status.WAITING)));
criteria.setCacheable(true);

2
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java
View File

@ -56,6 +56,7 @@ import com.google.common.base.Preconditions;
import io.onedev.commons.launcher.loader.Listen;
import io.onedev.commons.launcher.loader.ListenerRegistry;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.MatrixRunner;
import io.onedev.commons.utils.concurrent.Prioritized;
import io.onedev.server.OneDev;
import io.onedev.server.OneException;
@ -123,7 +124,6 @@ import io.onedev.server.search.entity.pullrequest.PullRequestQuery;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.security.permission.ProjectPermission;
import io.onedev.server.security.permission.ProjectPrivilege;
import io.onedev.server.util.MatrixRunner;
import io.onedev.server.util.PullRequestConstants;
import io.onedev.server.util.facade.ProjectFacade;
import io.onedev.server.util.facade.UserFacade;

11
server-core/src/main/java/io/onedev/server/event/build/BuildPending.java Normal file
View File

@ -0,0 +1,11 @@
package io.onedev.server.event.build;
import io.onedev.server.model.Build;
public class BuildPending extends BuildEvent {
public BuildPending(Build build) {
super(null, build.getPendingDate(), build);
}
}

11
server-core/src/main/java/io/onedev/server/event/build/BuildQueueing.java
View File

@ -1,11 +0,0 @@
package io.onedev.server.event.build;
import io.onedev.server.model.Build;
public class BuildQueueing extends BuildEvent {
public BuildQueueing(Build build) {
super(null, build.getQueueingDate(), build);
}
}

14
server-core/src/main/java/io/onedev/server/model/Build.java
View File

@ -61,7 +61,7 @@ import io.onedev.server.web.editable.PropertyDescriptor;
indexes={@Index(columnList="o_project_id"), @Index(columnList="o_submitter_id"), @Index(columnList="o_canceller_id"),
@Index(columnList="submitterName"), @Index(columnList="cancellerName"), @Index(columnList="commitHash"),
@Index(columnList="number"), @Index(columnList="jobName"), @Index(columnList="status"),
@Index(columnList="submitDate"), @Index(columnList="queueingDate"), @Index(columnList="runningDate"),
@Index(columnList="submitDate"), @Index(columnList="pendingDate"), @Index(columnList="runningDate"),
@Index(columnList="finishDate"), @Index(columnList="version"),
@Index(columnList="o_project_id, commitHash")},
uniqueConstraints={@UniqueConstraint(columnNames={"o_project_id", "number"})}
@ -79,7 +79,7 @@ public class Build extends AbstractEntity implements Referenceable {
public enum Status {
// Most significant status comes first, refer to getOverallStatus
WAITING, QUEUEING, RUNNING, IN_ERROR, FAILED, CANCELLED, TIMED_OUT, SUCCESSFUL;
WAITING, PENDING, RUNNING, IN_ERROR, FAILED, CANCELLED, TIMED_OUT, SUCCESSFUL;
public String getDisplayName() {
return StringUtils.capitalize(name().replace('_', ' ').toLowerCase());
@ -126,7 +126,7 @@ public class Build extends AbstractEntity implements Referenceable {
@Column(nullable=false)
private Date submitDate;
private Date queueingDate;
private Date pendingDate;
private Date runningDate;
@ -266,12 +266,12 @@ public class Build extends AbstractEntity implements Referenceable {
this.submitDate = submitDate;
}
public Date getQueueingDate() {
return queueingDate;
public Date getPendingDate() {
return pendingDate;
}
public void setQueueingDate(Date queueingDate) {
this.queueingDate = queueingDate;
public void setPendingDate(Date pendingDate) {
this.pendingDate = pendingDate;
}
public Date getRunningDate() {

2
server-core/src/main/java/io/onedev/server/model/Project.java
View File

@ -71,6 +71,7 @@ import io.onedev.commons.launcher.loader.ListenerRegistry;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.LinearRange;
import io.onedev.commons.utils.LockUtils;
import io.onedev.commons.utils.MatrixRunner;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.stringmatch.ChildAwareMatcher;
import io.onedev.commons.utils.stringmatch.Matcher;
@ -120,7 +121,6 @@ import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.security.permission.DefaultPrivilege;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.MatrixRunner;
import io.onedev.server.util.facade.ProjectFacade;
import io.onedev.server.util.jackson.DefaultView;
import io.onedev.server.util.patternset.PatternSet;

81
server-core/src/main/java/io/onedev/server/model/support/JobExecutionContext.java Normal file
View File

@ -0,0 +1,81 @@
package io.onedev.server.model.support;
import java.io.File;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.util.patternset.PatternSet;
public abstract class JobExecutionContext {
private final String environment;
private final File workspace;
private final Map<String, String> envVars;
private final List<String> commands;
private final @Nullable SourceSnapshot snapshot;
private final Collection<JobCache> caches;
private final PatternSet collectFiles;
private final Logger logger;
public JobExecutionContext(String environment, File workspace, Map<String, String> envVars,
List<String> commands, @Nullable SourceSnapshot snapshot, Collection<JobCache> caches,
PatternSet collectFiles, Logger logger) {
this.environment = environment;
this.workspace = workspace;
this.envVars = envVars;
this.commands = commands;
this.snapshot = snapshot;
this.caches = caches;
this.collectFiles = collectFiles;
this.logger = logger;
}
public String getEnvironment() {
return environment;
}
public File getWorkspace() {
return workspace;
}
public Map<String, String> getEnvVars() {
return envVars;
}
public List<String> getCommands() {
return commands;
}
@Nullable
public SourceSnapshot getSnapshot() {
return snapshot;
}
public Collection<JobCache> getCaches() {
return caches;
}
public PatternSet getCollectFiles() {
return collectFiles;
}
public Logger getLogger() {
return logger;
}
public abstract void notifyJobRunning();
}

15
server-core/src/main/java/io/onedev/server/model/support/JobExecutor.java
View File

@ -2,20 +2,13 @@ package io.onedev.server.model.support;
import java.io.File;
import java.io.Serializable;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import org.eclipse.jgit.lib.ObjectId;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import io.onedev.commons.launcher.loader.ExtensionPoint;
import io.onedev.commons.utils.stringmatch.ChildAwareMatcher;
import io.onedev.commons.utils.stringmatch.Matcher;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.model.Project;
import io.onedev.server.util.Usage;
import io.onedev.server.util.patternset.PatternSet;
@ -127,9 +120,7 @@ public abstract class JobExecutor implements Serializable {
this.cacheTTL = cacheTTL;
}
public abstract void execute(String environment, File workspace, Map<String, String> envVars,
List<String> commands, @Nullable SourceSnapshot snapshot, Collection<JobCache> caches,
PatternSet collectFiles, Logger logger);
public abstract void execute(JobExecutionContext context);
public final boolean isApplicable(Project project, ObjectId commitId, String jobName, String environment) {
Matcher matcher = new ChildAwareMatcher();
@ -160,10 +151,6 @@ public abstract class JobExecutor implements Serializable {
setProjects(patternSet.toString());
}
public boolean hasCapacity() {
return true;
}
public abstract void checkCaches();
public abstract void cleanDir(File dir);

2
server-core/src/main/java/io/onedev/server/search/entity/EntityCriteria.java
View File

@ -11,10 +11,10 @@ import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.utils.RangeBuilder;
import io.onedev.server.model.AbstractEntity;
import io.onedev.server.model.Project;
import io.onedev.server.model.User;
import io.onedev.server.util.RangeBuilder;
public abstract class EntityCriteria<T extends AbstractEntity> implements Serializable {

6
server-core/src/main/java/io/onedev/server/search/entity/build/BuildQuery.g4
View File

@ -7,7 +7,7 @@ query
;
criteria
: operator=(Successful|Failed|InError|Cancelled|Running|Waiting|Queueing|TimedOut|SubmittedByMe|CancelledByMe) #OperatorCriteria
: operator=(Successful|Failed|InError|Cancelled|Running|Waiting|Pending|TimedOut|SubmittedByMe|CancelledByMe) #OperatorCriteria
| operator=(FixedIssue|SubmittedBy|CancelledBy|DependsOn|DependenciesOf|RequiredByPullRequest) WS+ criteriaValue=Quoted #OperatorValueCriteria
| criteriaField=Quoted WS+ operator=(Is|IsGreaterThan|IsLessThan|IsBefore|IsAfter) WS+ criteriaValue=Quoted #FieldOperatorValueCriteria
| criteria WS+ And WS+ criteria #AndCriteria
@ -52,8 +52,8 @@ Waiting
: 'waiting'
;
Queueing
: 'queueing'
Pending
: 'pending'
;
SubmittedByMe

4
server-core/src/main/java/io/onedev/server/search/entity/build/BuildQuery.java
View File

@ -112,8 +112,8 @@ public class BuildQuery extends EntityQuery<Build> {
return new InErrorCriteria();
case BuildQueryLexer.Waiting:
return new WaitingCriteria();
case BuildQueryLexer.Queueing:
return new QueueingCriteria();
case BuildQueryLexer.Pending:
return new PendingCriteria();
case BuildQueryLexer.Running:
return new RunningCriteria();
case BuildQueryLexer.SubmittedByMe:

8
server-core/src/main/java/io/onedev/server/search/entity/build/QueueingCriteria.java → server-core/src/main/java/io/onedev/server/search/entity/build/PendingCriteria.java
View File

@ -11,19 +11,19 @@ import io.onedev.server.model.User;
import io.onedev.server.search.entity.EntityCriteria;
import io.onedev.server.util.BuildConstants;
public class QueueingCriteria extends EntityCriteria<Build> {
public class PendingCriteria extends EntityCriteria<Build> {
private static final long serialVersionUID = 1L;
@Override
public Predicate getPredicate(Project project, Root<Build> root, CriteriaBuilder builder, User user) {
Path<?> attribute = root.get(BuildConstants.ATTR_STATUS);
return builder.equal(attribute, Build.Status.QUEUEING);
return builder.equal(attribute, Build.Status.PENDING);
}
@Override
public boolean matches(Build build, User user) {
return build.getStatus() == Build.Status.QUEUEING;
return build.getStatus() == Build.Status.PENDING;
}
@Override
@ -33,7 +33,7 @@ public class QueueingCriteria extends EntityCriteria<Build> {
@Override
public String toString() {
return BuildQuery.getRuleName(BuildQueryLexer.Queueing);
return BuildQuery.getRuleName(BuildQueryLexer.Pending);
}
}

4
server-core/src/main/java/io/onedev/server/search/entity/build/QueueingDateCriteria.java
View File

@ -41,9 +41,9 @@ public class QueueingDateCriteria extends EntityCriteria<Build> {
@Override
public boolean matches(Build build, User user) {
if (operator == BuildQueryLexer.IsBefore)
return build.getQueueingDate().before(value);
return build.getPendingDate().before(value);
else
return build.getQueueingDate().after(value);
return build.getPendingDate().after(value);
}
@Override

6
server-core/src/main/java/io/onedev/server/search/entity/pullrequest/PullRequestQuery.g4
View File

@ -7,7 +7,7 @@ query
;
criteria
: operator=(Open|Merged|Discarded|SubmittedByMe|ToBeReviewedByMe|RequestedForChangesByMe|ApprovedByMe|DiscardedByMe|SomeoneRequestedForChanges|HasPendingReviews|HasFailedBuilds|HasPendingBuilds|HasMergeConflicts) #OperatorCriteria
: operator=(Open|Merged|Discarded|SubmittedByMe|ToBeReviewedByMe|RequestedForChangesByMe|ApprovedByMe|DiscardedByMe|SomeoneRequestedForChanges|HasPendingReviews|HasFailedBuilds|ToBeVerifiedByBuilds|HasMergeConflicts) #OperatorCriteria
| operator=(ToBeReviewedBy|ApprovedBy|RequestedForChangesBy|SubmittedBy|DiscardedBy|IncludesCommit|IncludesIssue) WS+ criteriaValue=Quoted #OperatorValueCriteria
| criteriaField=Quoted WS+ operator=(Is|IsGreaterThan|IsLessThan|IsBefore|IsAfter|Contains) WS+ criteriaValue=Quoted #FieldOperatorValueCriteria
| criteria WS+ And WS+ criteria #AndCriteria
@ -68,8 +68,8 @@ HasFailedBuilds
: 'has' WS+ 'failed' WS+ 'builds'
;
HasPendingBuilds
: 'has' WS+ 'pending' WS+ 'builds'
ToBeVerifiedByBuilds
: 'to' WS+ 'be' WS+ 'verified' WS+ 'by' WS+ 'builds'
;
HasMergeConflicts

4
server-core/src/main/java/io/onedev/server/search/entity/pullrequest/PullRequestQuery.java
View File

@ -109,8 +109,8 @@ public class PullRequestQuery extends EntityQuery<PullRequest> {
return new HasFailedBuildsCriteria();
case PullRequestQueryLexer.HasMergeConflicts:
return new HasMergeConflictsCriteria();
case PullRequestQueryLexer.HasPendingBuilds:
return new HasPendingBuildsCriteria();
case PullRequestQueryLexer.ToBeVerifiedByBuilds:
return new ToBeVerifiedByBuildsCriteria();
case PullRequestQueryLexer.HasPendingReviews:
return new HasPendingReviewsCriteria();
default:

8
server-core/src/main/java/io/onedev/server/search/entity/pullrequest/HasPendingBuildsCriteria.java → server-core/src/main/java/io/onedev/server/search/entity/pullrequest/ToBeVerifiedByBuildsCriteria.java
View File

@ -14,7 +14,7 @@ import io.onedev.server.model.PullRequest;
import io.onedev.server.model.User;
import io.onedev.server.util.PullRequestConstants;
public class HasPendingBuildsCriteria extends PullRequestCriteria {
public class ToBeVerifiedByBuildsCriteria extends PullRequestCriteria {
private static final long serialVersionUID = 1L;
@ -28,7 +28,7 @@ public class HasPendingBuildsCriteria extends PullRequestCriteria {
return builder.or(
builder.equal(status, Build.Status.RUNNING),
builder.equal(status, Build.Status.QUEUEING),
builder.equal(status, Build.Status.PENDING),
builder.equal(status, Build.Status.WAITING));
}
@ -36,7 +36,7 @@ public class HasPendingBuildsCriteria extends PullRequestCriteria {
public boolean matches(PullRequest request, User user) {
for (PullRequestBuild build: request.getPullRequestBuilds()) {
if (build.getBuild().getStatus() == Build.Status.RUNNING
|| build.getBuild().getStatus() == Build.Status.QUEUEING
|| build.getBuild().getStatus() == Build.Status.PENDING
|| build.getBuild().getStatus() == Build.Status.WAITING) {
return true;
}
@ -51,7 +51,7 @@ public class HasPendingBuildsCriteria extends PullRequestCriteria {
@Override
public String toString() {
return PullRequestQuery.getRuleName(PullRequestQueryLexer.HasPendingBuilds);
return PullRequestQuery.getRuleName(PullRequestQueryLexer.ToBeVerifiedByBuilds);
}
}

29
server-core/src/main/java/io/onedev/server/util/Maps.java
View File

@ -1,29 +0,0 @@
package io.onedev.server.util;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.common.base.Preconditions;
public class Maps {
public static <T> Map<T, T> newLinkedHashMap(@SuppressWarnings("unchecked") T...args) {
Map<T, T> map = new LinkedHashMap<>();
initMap(map, args);
return map;
}
public static <T> Map<T, T> newHashMap(@SuppressWarnings("unchecked") T...args) {
Map<T, T> map = new HashMap<>();
initMap(map, args);
return map;
}
private static <T> void initMap(Map<T, T> map, @SuppressWarnings("unchecked") T...args) {
Preconditions.checkArgument(args.length % 2 == 0, "Arguments should be key/value pairs");
for (int i=0; i<args.length/2; i++)
map.put(args[i*2], args[i*2+1]);
}
}

46
server-core/src/main/java/io/onedev/server/util/MatrixRunner.java
View File

@ -1,46 +0,0 @@
package io.onedev.server.util;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public abstract class MatrixRunner<T> {
private final Map<String, T> paramMap;
private final Map<String, List<T>> paramMatrix;
public MatrixRunner(Map<String, List<T>> paramMatrix) {
this(new LinkedHashMap<>(), paramMatrix);
}
private MatrixRunner(Map<String, T> paramMap, Map<String, List<T>> paramMatrix) {
this.paramMap = paramMap;
this.paramMatrix = paramMatrix;
}
public void run() {
if (!paramMatrix.isEmpty()) {
Map.Entry<String, List<T>> entry = paramMatrix.entrySet().iterator().next();
for (T value: entry.getValue()) {
Map<String, T> paramsCopy = new LinkedHashMap<>(paramMap);
paramsCopy.put(entry.getKey(), value);
Map<String, List<T>> matrixCopy = new LinkedHashMap<>(paramMatrix);
matrixCopy.remove(entry.getKey());
new MatrixRunner<T>(paramsCopy, matrixCopy) {
@Override
protected void run(Map<String, T> paramMap) {
MatrixRunner.this.run(paramMap);
}
}.run();
}
} else {
run(paramMap);
}
}
protected abstract void run(Map<String, T> paramMap);
}

37
server-core/src/main/java/io/onedev/server/util/RangeBuilder.java
View File

@ -1,37 +0,0 @@
package io.onedev.server.util;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class RangeBuilder {
private final List<List<Long>> ranges = new ArrayList<>();
public RangeBuilder(List<Long> discreteValues, List<Long> allValues) {
Map<Long, Integer> indexes = new HashMap<>();
for (int i=0; i<allValues.size(); i++)
indexes.put(allValues.get(i), i);
List<Long> continuousValues = new ArrayList<>();
int lastIndex = -1;
for (Long value: discreteValues) {
Integer index = indexes.get(value);
if (index != null) {
if (lastIndex != -1 && index - lastIndex > 1) {
ranges.add(continuousValues);
continuousValues = new ArrayList<>();
}
continuousValues.add(value);
lastIndex = index;
}
}
if (!continuousValues.isEmpty())
ranges.add(continuousValues);
}
public List<List<Long>> getRanges() {
return ranges;
}
}

6
server-core/src/main/java/io/onedev/server/web/component/build/side/BuildSidePanel.java
View File

@ -143,9 +143,9 @@ public abstract class BuildSidePanel extends Panel {
protected String load() {
long duration;
if (getBuild().getRunningDate() != null)
duration = getBuild().getRunningDate().getTime() - getBuild().getQueueingDate().getTime();
duration = getBuild().getRunningDate().getTime() - getBuild().getPendingDate().getTime();
else
duration = System.currentTimeMillis() - getBuild().getQueueingDate().getTime();
duration = System.currentTimeMillis() - getBuild().getPendingDate().getTime();
if (duration < 0)
duration = 0;
return DateUtils.formatDuration(duration);
@ -156,7 +156,7 @@ public abstract class BuildSidePanel extends Panel {
@Override
protected void onConfigure() {
super.onConfigure();
setVisible(getBuild().getQueueingDate() != null
setVisible(getBuild().getPendingDate() != null
&& (!getBuild().isFinished() || getBuild().getRunningDate() != null));
}

6
server-core/src/main/java/io/onedev/server/web/component/build/status/BuildStatusIcon.java
View File

@ -36,7 +36,7 @@ public class BuildStatusIcon extends GenericPanel<Status> {
super.onComponentTag(tag);
Status status = getModelObject();
String cssClass = "fa fa-fw build-status build-status-";
String cssClass = "fa build-status build-status-";
if (status != null)
cssClass += status.name().toLowerCase();
else
@ -87,8 +87,8 @@ public class BuildStatusIcon extends GenericPanel<Status> {
String title;
if (status == Status.WAITING)
title = "Waiting for completion of dependency builds";
else if (status == Status.QUEUEING)
title = "Queued due to limited capacity";
else if (status == Status.PENDING)
title = "Pending (preparing environments and/or waiting for resources)";
else if (status != null)
title = StringUtils.capitalize(status.getDisplayName().toLowerCase());
else

4
server-core/src/main/java/io/onedev/server/web/component/build/status/build-status.css
View File

@ -25,7 +25,7 @@
.build-status-waiting:before {
content: "\f28c";
}
.build-status-queueing:before {
.build-status-pending:before {
content: "\f192";
}
.build-status-none:before {
@ -37,7 +37,7 @@
.build-status-failed, .build-status-in_error, .build-status-cancelled, .build-status-timed_out {
color: red !important;
}
.build-status-running, .build-status-waiting, .build-status-queueing {
.build-status-running, .build-status-waiting, .build-status-pending {
color: #D2A906 !important;
}
.build-status-none {

12
server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java
View File

@ -76,7 +76,7 @@ public abstract class TaskButton extends AjaxButton {
}
})));
if (future != null)
if (future != null && !future.isDone())
future.cancel(true);
new ModalPanel(target) {
@ -127,7 +127,7 @@ public abstract class TaskButton extends AjaxButton {
@Override
protected void onCancel(AjaxRequestTarget target) {
Future<?> future = getTaskFutures().remove(path);
if (future != null)
if (future != null && !future.isDone())
future.cancel(true);
}
@ -180,8 +180,12 @@ public abstract class TaskButton extends AjaxButton {
@Override
public void execute() {
for (Iterator<Map.Entry<String, TaskFuture>> it = taskFutures.entrySet().iterator(); it.hasNext();) {
if (it.next().getValue().isExpired())
TaskFuture taskFuture = it.next().getValue();
if (taskFuture.isTimedout()) {
if (!taskFuture.isDone())
taskFuture.cancel(true);
it.remove();
}
}
}
@ -217,7 +221,7 @@ public abstract class TaskButton extends AjaxButton {
return wrapped.isDone();
}
public boolean isExpired() {
public boolean isTimedout() {
return timestamp.before(new DateTime().minusHours(1).toDate());
}

2
server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java
View File

@ -306,7 +306,7 @@ public abstract class BuildDetailPage extends ProjectPage implements InputContex
case SUCCESSFUL:
return "alert-success";
case WAITING:
case QUEUEING:
case PENDING:
case RUNNING:
return "alert-warning";
default:

2
server-core/src/test/java/io/onedev/server/util/MatrixRunnerTest.java
View File

@ -13,6 +13,8 @@ import org.junit.Test;
import com.google.common.collect.Lists;
import io.onedev.commons.utils.MatrixRunner;
public class MatrixRunnerTest {
@Test

2
server-core/src/test/java/io/onedev/server/util/RangeBuilderTest.java
View File

@ -8,6 +8,8 @@ import org.junit.Test;
import com.google.common.collect.Lists;
import io.onedev.commons.utils.RangeBuilder;
public class RangeBuilderTest {
@Test

15
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
View File

@ -6,7 +6,6 @@ import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -24,18 +23,17 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.Maps;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.ExecuteResult;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.server.OneException;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.model.support.JobExecutionContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.model.support.SourceSnapshot;
import io.onedev.server.plugin.kubernetes.KubernetesExecutor.TestData;
import io.onedev.server.util.Maps;
import io.onedev.server.util.patternset.PatternSet;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.OmitName;
import io.onedev.server.web.util.Testable;
@ -127,9 +125,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
}
@Override
public void execute(String environment, File workspace, Map<String, String> envVars,
List<String> commands, SourceSnapshot snapshot, Collection<JobCache> caches,
PatternSet collectFiles, Logger logger) {
public void execute(JobExecutionContext context) {
}
@Override
@ -392,7 +388,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
throw new OneException("Unexpected end of pod event watching");
} catch (Exception e) {
if (e.getCause() instanceof InterruptedException) {
if (ExceptionUtils.find(e, InterruptedException.class) != null) {
if (podStartedRef.get()) {
kubectl = newKubeCtl();
kubectl.addArgs("logs", podName, "-n", getNamespace(), "--follow");
@ -420,6 +416,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
try {
Thread.sleep(1000);
} catch (InterruptedException e2) {
throw new RuntimeException(e2);
}
} else {
result.checkReturnCode();

56
server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java
View File

@ -32,16 +32,15 @@ import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.commons.utils.command.ProcessKiller;
import io.onedev.commons.utils.concurrent.ConstrainedRunner;
import io.onedev.commons.utils.concurrent.CapacityRunner;
import io.onedev.server.ci.job.cache.CacheAllocation;
import io.onedev.server.ci.job.cache.CacheCallable;
import io.onedev.server.ci.job.cache.CacheRunner;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.model.support.JobExecutionContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.model.support.SourceSnapshot;
import io.onedev.server.plugin.serverdocker.ServerDockerExecutor.TestData;
import io.onedev.server.util.OneContext;
import io.onedev.server.util.patternset.PatternSet;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@ -70,7 +69,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
private List<RegistryLogin> registryLogins = new ArrayList<>();
private transient ConstrainedRunner constrainedRunner;
private transient CapacityRunner capacityRunner;
@Editable(order=20000, group="More Settings", description="Optionally specify docker executable, for instance <i>/usr/local/bin/docker</i>. "
+ "Leave empty to use docker executable in PATH")
@ -149,15 +148,10 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
}
private synchronized ConstrainedRunner getConstrainedRunner() {
if (constrainedRunner == null)
constrainedRunner = new ConstrainedRunner(capacity);
return constrainedRunner;
}
@Override
public boolean hasCapacity() {
return getConstrainedRunner().hasCapacity();
private synchronized CapacityRunner getCapacityRunner() {
if (capacityRunner == null)
capacityRunner = new CapacityRunner(capacity);
return capacityRunner;
}
private File getCacheHome() {
@ -165,33 +159,33 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
@Override
public void execute(String environment, File workspace, Map<String, String> envVars,
List<String> commands, SourceSnapshot snapshot, Collection<JobCache> caches,
PatternSet collectFiles, Logger logger) {
getConstrainedRunner().call(new Callable<Void>() {
public void execute(JobExecutionContext context) {
getCapacityRunner().call(new Callable<Void>() {
@Override
public Void call() {
return new CacheRunner(getCacheHome(), caches).call(new CacheCallable<Void>() {
return new CacheRunner(getCacheHome(), context.getCaches()).call(new CacheCallable<Void>() {
@Override
public Void call(Collection<CacheAllocation> allocations) {
context.notifyJobRunning();
login(logger);
logger.info("Pulling image...") ;
Commandline docker = getDocker();
docker.addArgs("pull", environment);
docker.addArgs("pull", context.getEnvironment());
docker.execute(newInfoLogger(logger), newErrorLogger(logger)).checkReturnCode();
docker.clearArgs();
String jobInstance = UUID.randomUUID().toString();
docker.addArgs("run", "--rm", "--name", jobInstance);
for (Map.Entry<String, String> entry: envVars.entrySet())
for (Map.Entry<String, String> entry: context.getEnvVars().entrySet())
docker.addArgs("--env", entry.getKey() + "=" + entry.getValue());
if (getRunOptions() != null)
docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
String imageOS = getImageOS(logger, environment);
String imageOS = getImageOS(logger, context.getEnvironment());
logger.info("Detected image OS: " + imageOS);
boolean windows = imageOS.equals("windows");
@ -210,16 +204,16 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
}
File effectiveWorkspace = workspaceCache != null? workspaceCache: workspace;
File effectiveWorkspace = workspaceCache != null? workspaceCache: context.getWorkspace();
if (snapshot != null) {
if (context.getSnapshot() != null) {
logger.info("Cloning source code...");
snapshot.checkout(effectiveWorkspace);
context.getSnapshot().checkout(effectiveWorkspace);
}
if (workspaceCache != null) {
try {
FileUtils.copyDirectory(workspace, workspaceCache);
FileUtils.copyDirectory(context.getWorkspace(), workspaceCache);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -235,20 +229,20 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
if (windows) {
File scriptFile = new File(effectiveWorkspace, "onedev-job-commands.bat");
try {
FileUtils.writeLines(scriptFile, commands, "\r\n");
FileUtils.writeLines(scriptFile, context.getCommands(), "\r\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
docker.addArgs(environment);
docker.addArgs(context.getEnvironment());
docker.addArgs("cmd", "/c", dockerWorkspacePath + "\\onedev-job-commands.bat");
} else {
File scriptFile = new File(effectiveWorkspace, "onedev-job-commands.sh");
try {
FileUtils.writeLines(scriptFile, commands, "\n");
FileUtils.writeLines(scriptFile, context.getCommands(), "\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
docker.addArgs(environment);
docker.addArgs(context.getEnvironment());
docker.addArgs("sh", dockerWorkspacePath + "/onedev-job-commands.sh");
}
@ -271,9 +265,9 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
} finally {
if (workspaceCache != null) {
int baseLen = workspaceCache.getAbsolutePath().length()+1;
for (File file: collectFiles.listFiles(workspaceCache)) {
for (File file: context.getCollectFiles().listFiles(workspaceCache)) {
try {
FileUtils.copyFile(file, new File(workspace, file.getAbsolutePath().substring(baseLen)));
FileUtils.copyFile(file, new File(context.getWorkspace(), file.getAbsolutePath().substring(baseLen)));
} catch (IOException e) {
throw new RuntimeException(e);
}