archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

A basic implementation of Kubernetes executor

Browse Source
This commit is contained in:
Robin Shen 2019-06-26 09:04:55 +08:00
parent 854c920f9d
commit cf9d8c86e7
63 changed files with 762 additions and 919 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -175,7 +175,7 @@
<groupId>io.onedev</groupId>
<artifactId>commons-jsyntax</artifactId>
<version>${commons.version}</version>
</dependency>
</dependency>
</dependencies>
</dependencyManagement>
<repositories>
@ -195,7 +195,7 @@
</repository>
</repositories>
<properties>
<commons.version>1.1.2</commons.version>
<commons.version>1.1.3</commons.version>
<antlr.version>4.7.2</antlr.version>
</properties>
</project>

8
server-core/src/main/java/io/onedev/server/CoreModule.java
View File

@ -84,9 +84,8 @@ import io.onedev.server.ci.DefaultCISpecProvider;
import io.onedev.server.ci.job.DefaultJobManager;
import io.onedev.server.ci.job.DependencyPopulator;
import io.onedev.server.ci.job.JobManager;
import io.onedev.server.ci.job.log.DefaultLogManager;
import io.onedev.server.ci.job.log.LogManager;
import io.onedev.server.ci.job.log.LogNormalizer;
import io.onedev.server.ci.job.log.DefaultJobLogManager;
import io.onedev.server.ci.job.log.JobLogManager;
import io.onedev.server.ci.job.log.instruction.LogInstruction;
import io.onedev.server.entitymanager.BuildDependenceManager;
import io.onedev.server.entitymanager.BuildManager;
@ -318,7 +317,7 @@ public class CoreModule extends AbstractPluginModule {
bind(BuildManager.class).to(DefaultBuildManager.class);
bind(BuildDependenceManager.class).to(DefaultBuildDependenceManager.class);
bind(JobManager.class).to(DefaultJobManager.class);
bind(LogManager.class).to(DefaultLogManager.class);
bind(JobLogManager.class).to(DefaultJobLogManager.class);
bind(PullRequestBuildManager.class).to(DefaultPullRequestBuildManager.class);
bind(MailManager.class).to(DefaultMailManager.class);
bind(IssueManager.class).to(DefaultIssueManager.class);
@ -389,7 +388,6 @@ public class CoreModule extends AbstractPluginModule {
contributeFromPackage(Authenticator.class, Authenticator.class);
contributeFromPackage(DefaultCISpecProvider.class, DefaultCISpecProvider.class);
contributeFromPackage(LogNormalizer.class, LogNormalizer.class);
contribute(CodePullAuthorizationSource.class, DefaultJobManager.class);

6
server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java vendored
View File

@ -448,14 +448,14 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
revList.revisions(revisions).order(Order.TOPO);
List<ObjectId> historyIds = new ArrayList<>();
for (String commitHash: revList.call(null))
for (String commitHash: revList.call())
historyIds.add(ObjectId.fromString(commitHash));
revList = new RevListCommand(project.getGitDir());
revList.order(null).firstParent(true);
Set<ObjectId> firstParentIds = new HashSet<>();
for (String commitHash: revList.call(null))
for (String commitHash: revList.call())
firstParentIds.add(ObjectId.fromString(commitHash));
/*
@ -565,7 +565,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
@Override
public void run() {
try {
log.call(null);
log.call();
} catch (Exception e) {
logException.set(e);
} finally {

63
server-core/src/main/java/io/onedev/server/ci/job/DefaultJobManager.java
View File

@ -15,6 +15,7 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
@ -47,7 +48,8 @@ import io.onedev.server.OneException;
import io.onedev.server.ci.CISpec;
import io.onedev.server.ci.InvalidCISpecException;
import io.onedev.server.ci.JobDependency;
import io.onedev.server.ci.job.log.LogManager;
import io.onedev.server.ci.job.log.JobLogManager;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.ci.job.param.JobParam;
import io.onedev.server.ci.job.trigger.JobTrigger;
import io.onedev.server.entitymanager.BuildManager;
@ -104,7 +106,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
private final SessionManager sessionManager;
private final LogManager logManager;
private final JobLogManager logManager;
private final UserManager userManager;
@ -127,7 +129,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
@Inject
public DefaultJobManager(BuildManager buildManager, UserManager userManager,
ListenerRegistry listenerRegistry, SettingManager settingManager,
TransactionManager transactionManager, LogManager logManager, ExecutorService executorService,
TransactionManager transactionManager, JobLogManager logManager, ExecutorService executorService,
SessionManager sessionManager, Set<DependencyPopulator> dependencyPopulators,
TaskScheduler taskScheduler, BuildParamManager buildParamManager) {
this.settingManager = settingManager;
@ -248,21 +250,22 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
private void execute(Build build) {
try {
String jobId = UUID.randomUUID().toString();
Collection<String> jobSecretsToMask = Sets.newHashSet(jobId);
String jobToken = UUID.randomUUID().toString();
Collection<String> jobSecretsToMask = Sets.newHashSet(jobToken);
Job job = build.getJob();
ObjectId commitId = ObjectId.fromString(build.getCommitHash());
JobExecutor executor = getJobExecutor(build.getProject(), commitId, job.getName(), job.getEnvironment());
if (executor != null) {
Logger logger = logManager.getLogger(build, job.getLogLevel(), jobSecretsToMask);
JobLogger logger = logManager.getLogger(build, jobSecretsToMask);
Long buildId = build.getId();
String projectName = build.getProject().getName();
File projectGitDir = build.getProject().getGitDir();
JobExecution execution = new JobExecution(executorService.submit(new Runnable() {
@Override
public void run() {
logger.info("Creating server workspace...");
logger.log("Creating server workspace...");
File serverWorkspace = FileUtils.createTempDir("server-workspace");
try {
Map<String, String> envVars = new HashMap<>();
@ -274,7 +277,7 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
@Override
public void run() {
Build build = buildManager.load(buildId);
logger.info("Populating dependencies...");
logger.log("Populating job dependencies...");
for (BuildDependence dependence: build.getDependencies()) {
for (DependencyPopulator populator: dependencyPopulators)
populator.populate(dependence.getDependency(), serverWorkspace, logger);
@ -313,12 +316,13 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
});
logger.info("Executing job with executor '" + executor.getName() + "'...");
logger.log("Executing job with executor '" + executor.getName() + "'...");
List<String> commands = Splitter.on("\n").trimResults(CharMatcher.is('\r')).splitToList(job.getCommands());
JobContext jobContext = new JobContext(projectName, job.getEnvironment(), serverWorkspace, envVars, commands,
job.isCloneSource(), commitId, job.getCaches(), new PatternSet(includeFiles, excludeFiles), logger) {
JobContext jobContext = new JobContext(projectName, projectGitDir, job.getEnvironment(),
serverWorkspace, envVars, commands, job.isCloneSource(), commitId, job.getCaches(),
new PatternSet(includeFiles, excludeFiles), logger) {
@Override
public void notifyJobRunning() {
@ -338,18 +342,18 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
};
jobContexts.put(jobId, jobContext);
jobContexts.put(jobToken, jobContext);
try {
executor.execute(jobId, jobContext);
executor.execute(jobToken, jobContext);
} finally {
jobContexts.remove(jobId);
jobContexts.remove(jobToken);
}
sessionManager.run(new Runnable() {
@Override
public void run() {
logger.info("Collecting job outcomes...");
logger.log("Processing job outcomes...");
Build build = buildManager.load(buildId);
for (JobOutcome outcome: job.getOutcomes())
outcome.process(build, serverWorkspace, logger);
@ -357,8 +361,11 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
});
} catch (Exception e) {
if (ExceptionUtils.find(e, InterruptedException.class) == null)
logger.error("Error running build", e);
if (ExceptionUtils.find(e, InterruptedException.class) == null) {
DefaultJobManager.logger.debug("Error running build", e);
if (e.getMessage() != null)
logger.log(e.getMessage());
}
String errorMessage = e.getMessage();
if (errorMessage != null) {
for (String secret: jobSecretsToMask)
@ -368,10 +375,10 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
throw e;
}
} finally {
logger.info("Deleting server workspace...");
logger.log("Deleting server workspace...");
executor.cleanDir(serverWorkspace);
FileUtils.deleteDir(serverWorkspace);
logger.info("Server workspace deleted");
logger.log("Job finished");
}
}
@ -390,8 +397,8 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
}
@Override
public JobContext getJobContext(String jobId) {
return jobContexts.get(jobId);
public JobContext getJobContext(String jobToken) {
return jobContexts.get(jobToken);
}
private void markBuildError(Build build, String errorMessage) {
@ -586,8 +593,12 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
build.setCanceller(userManager.load(cancellerId));
}
build.setStatus(Build.Status.CANCELLED);
} catch (Exception e) {
build.setStatus(Build.Status.FAILED, e.getMessage());
} catch (ExecutionException e) {
if (e.getCause() != null)
build.setStatus(Build.Status.FAILED, e.getCause().getMessage());
else
build.setStatus(Build.Status.FAILED, e.getMessage());
} catch (InterruptedException e) {
} finally {
build.setFinishDate(new Date());
listenerRegistry.post(new BuildFinished(build));
@ -628,9 +639,9 @@ public class DefaultJobManager implements JobManager, Runnable, SchedulableTask,
@Override
public boolean canPullCode(HttpServletRequest request, Project project) {
String jobId = request.getHeader(JOB_ID_HTTP_HEADER);
if (jobId != null) {
JobContext context = getJobContext(jobId);
String jobToken = request.getHeader(JOB_TOKEN_HTTP_HEADER);
if (jobToken != null) {
JobContext context = getJobContext(jobToken);
if (context != null)
return context.getProjectName().equals(project.getName());
}

5
server-core/src/main/java/io/onedev/server/ci/job/DependencyPopulator.java
View File

@ -2,12 +2,11 @@ package io.onedev.server.ci.job;
import java.io.File;
import org.slf4j.Logger;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.Build;
public interface DependencyPopulator {
void populate(Build dependency, File workspace, Logger logger);
void populate(Build dependency, File workspace, JobLogger logger);
}

12
server-core/src/main/java/io/onedev/server/ci/job/Job.java
View File

@ -21,7 +21,6 @@ import org.hibernate.validator.constraints.NotEmpty;
import io.onedev.server.ci.JobDependency;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.ci.job.log.LogLevel;
import io.onedev.server.ci.job.param.JobParam;
import io.onedev.server.ci.job.trigger.JobTrigger;
import io.onedev.server.event.ProjectEvent;
@ -61,8 +60,6 @@ public class Job implements Serializable, Validatable {
private long timeout = 3600;
private LogLevel logLevel = LogLevel.INFO;
private transient Map<String, InputSpec> paramSpecMap;
@Editable(order=100, description="Specify name of the job")
@ -168,15 +165,6 @@ public class Job implements Serializable, Validatable {
this.timeout = timeout;
}
@Editable(order=10300, group="More Settings")
public LogLevel getLogLevel() {
return logLevel;
}
public void setLogLevel(LogLevel logLevel) {
this.logLevel = logLevel;
}
public JobTrigger getMatchedTrigger(ProjectEvent event) {
for (JobTrigger trigger: getTriggers()) {
if (trigger.matches(event, this))

4
server-core/src/main/java/io/onedev/server/ci/job/JobManager.java
View File

@ -14,7 +14,7 @@ import io.onedev.server.model.support.JobContext;
public interface JobManager {
public static final String JOB_ID_HTTP_HEADER = "X-ONEDEV-JOB-ID";
public static final String JOB_TOKEN_HTTP_HEADER = "X-ONEDEV-JOB-TOKEN";
Build submit(Project project, ObjectId commitId, String jobName,
Map<String, List<String>> paramMap, @Nullable User submitter);
@ -23,6 +23,6 @@ public interface JobManager {
void cancel(Build build, @Nullable User canceller);
JobContext getJobContext(String jobId);
JobContext getJobContext(String jobToken);
}

4
server-core/src/main/java/io/onedev/server/ci/job/JobOutcome.java
View File

@ -4,9 +4,9 @@ import java.io.File;
import java.io.Serializable;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import io.onedev.server.OneDev;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.Build;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.patternset.PatternSet;
@ -39,7 +39,7 @@ public abstract class JobOutcome implements Serializable {
return patternSet;
}
public abstract void process(Build build, File workspace, Logger logger);
public abstract void process(Build build, File workspace, JobLogger logger);
public static String getLockKey(Build build, String outcomeDir) {
return "job-outcome:" + build.getId() + ":" + outcomeDir;

5
server-core/src/main/java/io/onedev/server/ci/job/cache/CacheRunner.java vendored
View File

@ -5,12 +5,15 @@ import java.util.ArrayList;
import java.util.Collection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
public class CacheRunner {
private static final Logger logger = LoggerFactory.getLogger(CacheRunner.class);
private final File cacheHome;
private final Collection<JobCache> caches;
@ -20,7 +23,7 @@ public class CacheRunner {
this.caches = caches;
}
public <T> T call(CacheCallable<T> callable, Logger logger) {
public <T> T call(CacheCallable<T> callable) {
Collection<CacheAllocation> allocations = new ArrayList<>();
try {
if (!cacheHome.exists())

123
server-core/src/main/java/io/onedev/server/ci/job/log/DefaultLogManager.java → server-core/src/main/java/io/onedev/server/ci/job/log/DefaultJobLogManager.java
View File

@ -16,7 +16,6 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.regex.Pattern;
@ -53,9 +52,9 @@ import io.onedev.server.util.inputspec.SecretInput;
import io.onedev.server.web.websocket.WebSocketManager;
@Singleton
public class DefaultLogManager implements LogManager {
public class DefaultJobLogManager implements JobLogManager {
private static final Logger logger = LoggerFactory.getLogger(DefaultLogManager.class);
private static final Logger logger = LoggerFactory.getLogger(DefaultJobLogManager.class);
private static final int MIN_CACHE_ENTRIES = 5000;
@ -73,17 +72,14 @@ public class DefaultLogManager implements LogManager {
private final BuildManager buildManager;
private final Set<LogNormalizer> logNormalizers;
private final Map<Long, LogSnippet> recentSnippets = new ConcurrentHashMap<>();
@Inject
public DefaultLogManager(StorageManager storageManager, WebSocketManager webSocketManager,
BuildManager buildManager, Set<LogNormalizer> logNormalizers) {
public DefaultJobLogManager(StorageManager storageManager, WebSocketManager webSocketManager,
BuildManager buildManager) {
this.storageManager = storageManager;
this.webSocketManager = webSocketManager;
this.buildManager = buildManager;
this.logNormalizers = logNormalizers;
}
private File getLogFile(Long projectId, Long buildNumber) {
@ -92,67 +88,53 @@ public class DefaultLogManager implements LogManager {
}
@Override
public Logger getLogger(Build build, LogLevel loggerLevel, Collection<String> jobSecretsToMask) {
public JobLogger getLogger(Build build, Collection<String> jobSecretsToMask) {
Long projectId = build.getProject().getId();
Long buildId = build.getId();
Long buildNumber = build.getNumber();
Collection<String> secretValuesToMask = build.getSecretValuesToMask();
secretValuesToMask.addAll(jobSecretsToMask);
return new JobLogger(loggerLevel) {
return new JobLogger() {
private static final long serialVersionUID = 1L;
private void log(LogLevel logLevel, String message) {
for (LogNormalizer logNormalizer: logNormalizers) {
LogNormalizer.Normalized normalized = logNormalizer.normalize(message);
if (normalized != null) {
if (normalized.getLevel() != null)
logLevel = normalized.getLevel();
message = normalized.getMessage();
break;
}
}
private void doLog(String message) {
for (String maskSecret: secretValuesToMask)
message = StringUtils.replace(message, maskSecret, SecretInput.MASK);
if (logLevel.ordinal() <= loggerLevel.ordinal()) {
Lock lock = LockUtils.getReadWriteLock(getLockKey(buildId)).writeLock();
lock.lock();
try {
LogSnippet snippet = recentSnippets.get(buildId);
if (snippet == null) {
File logFile = getLogFile(projectId, buildNumber);
if (!logFile.exists()) {
snippet = new LogSnippet();
recentSnippets.put(buildId, snippet);
}
Lock lock = LockUtils.getReadWriteLock(getLockKey(buildId)).writeLock();
lock.lock();
try {
LogSnippet snippet = recentSnippets.get(buildId);
if (snippet == null) {
File logFile = getLogFile(projectId, buildNumber);
if (!logFile.exists()) {
snippet = new LogSnippet();
recentSnippets.put(buildId, snippet);
}
if (snippet != null) {
snippet.entries.add(new LogEntry(new Date(), logLevel, message));
if (snippet.entries.size() > MAX_CACHE_ENTRIES) {
File logFile = getLogFile(projectId, buildNumber);
try (ObjectOutputStream oos = newOutputStream(logFile)) {
while (snippet.entries.size() > MIN_CACHE_ENTRIES) {
LogEntry entry = snippet.entries.remove(0);
oos.writeObject(entry);
snippet.offset++;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
webSocketManager.notifyObservableChange(Build.getLogWebSocketObservable(buildId), null);
}
} finally {
lock.unlock();
}
if (snippet != null) {
snippet.entries.add(new JobLogEntry(new Date(), message));
if (snippet.entries.size() > MAX_CACHE_ENTRIES) {
File logFile = getLogFile(projectId, buildNumber);
try (ObjectOutputStream oos = newOutputStream(logFile)) {
while (snippet.entries.size() > MIN_CACHE_ENTRIES) {
JobLogEntry entry = snippet.entries.remove(0);
oos.writeObject(entry);
snippet.offset++;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
webSocketManager.notifyObservableChange(Build.getLogWebSocketObservable(buildId), null);
}
} finally {
lock.unlock();
}
}
@Override
public void log(LogLevel logLevel, String message, Throwable throwable) {
public void log(String message, Throwable throwable) {
try {
if (throwable != null) {
for (String line: Splitter.on(EOL_PATTERN).split(Throwables.getStackTraceAsString(throwable)))
@ -160,7 +142,7 @@ public class DefaultLogManager implements LogManager {
}
if (message.startsWith(LogInstruction.PREFIX)) {
log(logLevel, message);
doLog(message);
InstructionContext instructionContext = LogInstruction.parse(message);
String name = instructionContext.Identifier().getText();
@ -186,13 +168,13 @@ public class DefaultLogManager implements LogManager {
paramValues.add(LogInstruction.unescape(LogInstruction.removeQuotes(terminalNode.getText())));
params.put(paramName, paramValues);
}
log(LogLevel.DEBUG, "Executing log instruction '" + name + "'...");
doLog("Executing log instruction '" + name + "'...");
doInSession(instruction, buildId, params);
} else {
log(LogLevel.ERROR, "Unsupported log instruction: " + name);
doLog("Unsupported log instruction: " + name);
}
} else {
log(logLevel, message);
doLog(message);
}
} catch (Exception e) {
logger.error("Error logging", e);
@ -211,8 +193,8 @@ public class DefaultLogManager implements LogManager {
return "build-log: " + buildId;
}
private List<LogEntry> readLogEntries(File logFile, int from, int count) {
List<LogEntry> entries = new ArrayList<>();
private List<JobLogEntry> readLogEntries(File logFile, int from, int count) {
List<JobLogEntry> entries = new ArrayList<>();
if (logFile.exists()) {
try (ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(logFile)))) {
int numOfReadEntries = 0;
@ -221,7 +203,7 @@ public class DefaultLogManager implements LogManager {
numOfReadEntries++;
}
while (count == 0 || numOfReadEntries - from < count) {
entries.add((LogEntry) ois.readObject());
entries.add((JobLogEntry) ois.readObject());
numOfReadEntries++;
}
} catch (EOFException e) {
@ -237,7 +219,7 @@ public class DefaultLogManager implements LogManager {
if (logFile.exists()) {
try (ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(logFile)))) {
while (true) {
snippet.entries.add((LogEntry) ois.readObject());
snippet.entries.add((JobLogEntry) ois.readObject());
if (snippet.entries.size() > count) {
snippet.entries.remove(0);
snippet.offset ++;
@ -251,7 +233,7 @@ public class DefaultLogManager implements LogManager {
return snippet;
}
private List<LogEntry> readLogEntries(List<LogEntry> cachedEntries, int from, int count) {
private List<JobLogEntry> readLogEntries(List<JobLogEntry> cachedEntries, int from, int count) {
if (from < cachedEntries.size()) {
int to = from + count;
if (to == from || to > cachedEntries.size())
@ -264,7 +246,7 @@ public class DefaultLogManager implements LogManager {
@Sessional
@Override
public List<LogEntry> readLogEntries(Build build, int from, int count) {
public List<JobLogEntry> readLogEntries(Build build, int from, int count) {
Lock lock = LockUtils.getReadWriteLock(getLockKey(build.getId())).readLock();
lock.lock();
try {
@ -274,7 +256,7 @@ public class DefaultLogManager implements LogManager {
if (from >= snippet.offset) {
return readLogEntries(snippet.entries, from - snippet.offset, count);
} else {
List<LogEntry> entries = new ArrayList<>();
List<JobLogEntry> entries = new ArrayList<>();
entries.addAll(readLogEntries(logFile, from, count));
if (count == 0)
entries.addAll(snippet.entries);
@ -347,7 +329,7 @@ public class DefaultLogManager implements LogManager {
if (snippet != null) {
File logFile = getLogFile(build.getProject().getId(), build.getNumber());
try (ObjectOutputStream oos = newOutputStream(logFile)) {
for (LogEntry entry: snippet.entries)
for (JobLogEntry entry: snippet.entries)
oos.writeObject(entry);
} catch (IOException e) {
throw new RuntimeException(e);
@ -387,7 +369,7 @@ public class DefaultLogManager implements LogManager {
LogSnippet snippet = recentSnippets.get(build.getId());
if (snippet != null) {
StringBuilder builder = new StringBuilder();
for (LogEntry entry: snippet.entries)
for (JobLogEntry entry: snippet.entries)
builder.append(renderAsText(entry) + "\n");
recentBuffer = builder.toString().getBytes(Charsets.UTF_8);
}
@ -397,9 +379,8 @@ public class DefaultLogManager implements LogManager {
}
}
private String renderAsText(LogEntry entry) {
String prefix = DATE_FORMATTER.print(new DateTime(entry.getDate())) + " "
+ StringUtils.leftPad(entry.getLevel().name(), 5) + " ";
private String renderAsText(JobLogEntry entry) {
String prefix = DATE_FORMATTER.print(new DateTime(entry.getDate())) + " ";
StringBuilder builder = new StringBuilder();
for (String line: Splitter.on(EOL_PATTERN).split(entry.getMessage())) {
if (builder.length() == 0) {
@ -419,7 +400,7 @@ public class DefaultLogManager implements LogManager {
if (pos == buffer.length) {
if (ois != null) {
try {
buffer = (renderAsText((LogEntry) ois.readObject()) + "\n").getBytes(Charsets.UTF_8);
buffer = (renderAsText((JobLogEntry) ois.readObject()) + "\n").getBytes(Charsets.UTF_8);
} catch (EOFException e) {
IOUtils.closeQuietly(ois);
ois = null;

11
server-core/src/main/java/io/onedev/server/ci/job/log/LogEntry.java → server-core/src/main/java/io/onedev/server/ci/job/log/JobLogEntry.java
View File

@ -3,19 +3,16 @@ package io.onedev.server.ci.job.log;
import java.io.Serializable;
import java.util.Date;
public class LogEntry implements Serializable {
public class JobLogEntry implements Serializable {
private static final long serialVersionUID = 1L;
private final Date date;
private final LogLevel level;
private final String message;
public LogEntry(Date date, LogLevel level, String message) {
public JobLogEntry(Date date, String message) {
this.date = date;
this.level = level;
this.message = message;
}
@ -23,10 +20,6 @@ public class LogEntry implements Serializable {
return date;
}
public LogLevel getLevel() {
return level;
}
public String getMessage() {
return message;
}

8
server-core/src/main/java/io/onedev/server/ci/job/log/LogManager.java → server-core/src/main/java/io/onedev/server/ci/job/log/JobLogManager.java
View File

@ -4,13 +4,11 @@ import java.io.InputStream;
import java.util.Collection;
import java.util.List;
import org.slf4j.Logger;
import io.onedev.server.model.Build;
public interface LogManager {
public interface JobLogManager {
Logger getLogger(Build build, LogLevel logLevel, Collection<String> jobSecrets);
JobLogger getLogger(Build build, Collection<String> jobSecrets);
/**
* Read specified number of log entries from specified build, starting from specified index
@ -26,7 +24,7 @@ public interface LogManager {
* log entries. Number of entries may be less than required count if there is no
* enough log entries
*/
List<LogEntry> readLogEntries(Build build, int offset, int count);
List<JobLogEntry> readLogEntries(Build build, int offset, int count);
/**
* Read specified number of log entries starting from end of the log

226
server-core/src/main/java/io/onedev/server/ci/job/log/JobLogger.java
View File

@ -2,230 +2,12 @@ package io.onedev.server.ci.job.log;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.helpers.MarkerIgnoringBase;
import org.slf4j.helpers.MessageFormatter;
public abstract class JobLogger {
public abstract class JobLogger extends MarkerIgnoringBase implements Logger {
private static final long serialVersionUID = 1L;
private final LogLevel logLevel;
public abstract void log(String message, @Nullable Throwable t);
public JobLogger(LogLevel logLevel) {
this.logLevel = logLevel;
public void log(String message) {
log(message, null);
}
@Override
public String getName() {
return "Job Logger";
}
@Override
public boolean isTraceEnabled() {
return logLevel.compareTo(LogLevel.TRACE) >= 0;
}
@Override
public void trace(String msg) {
if (isTraceEnabled())
log(LogLevel.TRACE, msg, null);
}
@Override
public void trace(String format, Object arg) {
if (isTraceEnabled()) {
String msgStr = MessageFormatter.format(format, arg).getMessage();
log(LogLevel.TRACE, msgStr, null);
}
}
@Override
public void trace(String format, Object arg1, Object arg2) {
if (isTraceEnabled()) {
String msgStr = MessageFormatter.format(format, arg1, arg2).getMessage();
log(LogLevel.TRACE, msgStr, null);
}
}
@Override
public void trace(String msg, Throwable t) {
if (isTraceEnabled())
log(LogLevel.TRACE, msg, t);
}
@Override
public boolean isDebugEnabled() {
return logLevel.compareTo(LogLevel.DEBUG) >= 0;
}
@Override
public void debug(String msg) {
if (isDebugEnabled())
log(LogLevel.DEBUG, msg, null);
}
@Override
public void debug(String format, Object arg) {
if (isDebugEnabled()) {
String msgStr = MessageFormatter.format(format, arg).getMessage();
log(LogLevel.DEBUG, msgStr, null);
}
}
@Override
public void debug(String format, Object arg1, Object arg2) {
if (isDebugEnabled()) {
String msgStr = MessageFormatter.format(format, arg1, arg2).getMessage();
log(LogLevel.DEBUG, msgStr, null);
}
}
@Override
public void debug(String msg, Throwable t) {
if (isDebugEnabled())
log(LogLevel.DEBUG, msg, t);
}
@Override
public boolean isInfoEnabled() {
return logLevel.compareTo(LogLevel.INFO) >= 0;
}
@Override
public void info(String msg) {
if (isInfoEnabled())
log(LogLevel.INFO, msg, null);
}
@Override
public void info(String format, Object arg) {
if (isInfoEnabled()) {
String msgStr = MessageFormatter.format(format, arg).getMessage();
log(LogLevel.INFO, msgStr, null);
}
}
@Override
public void info(String format, Object arg1, Object arg2) {
if (isInfoEnabled()) {
String msgStr = MessageFormatter.format(format, arg1, arg2).getMessage();
log(LogLevel.INFO, msgStr, null);
}
}
@Override
public void info(String msg, Throwable t) {
if (isInfoEnabled())
log(LogLevel.INFO, msg, t);
}
@Override
public boolean isWarnEnabled() {
return logLevel.compareTo(LogLevel.WARN) >= 0;
}
@Override
public void warn(String msg) {
if (isWarnEnabled())
log(LogLevel.WARN, msg, null);
}
@Override
public void warn(String format, Object arg) {
if (isWarnEnabled()) {
String msgStr = MessageFormatter.format(format, arg).getMessage();
log(LogLevel.WARN, msgStr, null);
}
}
@Override
public void warn(String format, Object arg1, Object arg2) {
if (isWarnEnabled()) {
String msgStr = MessageFormatter.format(format, arg1, arg2).getMessage();
log(LogLevel.WARN, msgStr, null);
}
}
@Override
public void warn(String msg, Throwable t) {
if (isWarnEnabled())
log(LogLevel.WARN, msg, t);
}
@Override
public boolean isErrorEnabled() {
return logLevel.compareTo(LogLevel.ERROR) >= 0;
}
@Override
public void error(String msg) {
if (isErrorEnabled())
log(LogLevel.ERROR, msg, null);
}
@Override
public void error(String format, Object arg) {
if (isErrorEnabled()) {
String msgStr = MessageFormatter.format(format, arg).getMessage();
log(LogLevel.ERROR, msgStr, null);
}
}
@Override
public void error(String format, Object arg1, Object arg2) {
if (isErrorEnabled()) {
String msgStr = MessageFormatter.format(format, arg1, arg2).getMessage();
log(LogLevel.ERROR, msgStr, null);
}
}
@Override
public void error(String msg, Throwable t) {
if (isErrorEnabled())
log(LogLevel.ERROR, msg, t);
}
@Override
public void debug(String arg0, Object... arg1) {
if (isDebugEnabled()) {
String msgStr = MessageFormatter.arrayFormat(arg0, arg1).getMessage();
log(LogLevel.DEBUG, msgStr, null);
}
}
@Override
public void error(String arg0, Object... arg1) {
if (isErrorEnabled()) {
String msgStr = MessageFormatter.arrayFormat(arg0, arg1).getMessage();
log(LogLevel.ERROR, msgStr, null);
}
}
@Override
public void info(String arg0, Object... arg1) {
if (isInfoEnabled()) {
String msgStr = MessageFormatter.arrayFormat(arg0, arg1).getMessage();
log(LogLevel.INFO, msgStr, null);
}
}
@Override
public void trace(String arg0, Object... arg1) {
if (isTraceEnabled()) {
String msgStr = MessageFormatter.arrayFormat(arg0, arg1).getMessage();
log(LogLevel.TRACE, msgStr, null);
}
}
@Override
public void warn(String arg0, Object... arg1) {
if (isWarnEnabled()) {
String msgStr = MessageFormatter.arrayFormat(arg0, arg1).getMessage();
log(LogLevel.WARN, msgStr, null);
}
}
protected abstract void log(LogLevel logLevel, String message, @Nullable Throwable throwable);
}

63
server-core/src/main/java/io/onedev/server/ci/job/log/LogNormalizer.java
View File

@ -1,63 +0,0 @@
package io.onedev.server.ci.job.log;
import javax.annotation.Nullable;
import io.onedev.commons.launcher.loader.ExtensionPoint;
/**
* Sometimes job log message needs to be normalized for better display. For instance Maven command prints something
* like below:
*
* <pre>[INFO] Scanning for projects...</pre>
*
* In such case, we should extract the log level information to override OneDev's default log level, and the
* original message should also be modified to remove the log level information
*
* @author robin
*
*/
@ExtensionPoint
public interface LogNormalizer {
/**
* Normalize provided job log message
* @param message
* message to be normalized
* @return
* normalized result, or <tt>null</tt> if this normalizer does not handle this message
*/
@Nullable
Normalized normalize(String message);
public static class Normalized {
private final LogLevel level;
private final String message;
public Normalized(@Nullable LogLevel level, String message) {
this.level = level;
this.message = message;
}
/**
* @return
* Log level of this message, or <tt>null</tt> if log level information
* is not available in this message
*/
@Nullable
public LogLevel getLevel() {
return level;
}
/**
* @return
* normalized message
*/
public String getMessage() {
return message;
}
}
}

2
server-core/src/main/java/io/onedev/server/ci/job/log/LogSnippet.java
View File

@ -5,7 +5,7 @@ import java.util.List;
public class LogSnippet {
public List<LogEntry> entries = new LinkedList<>();
public List<JobLogEntry> entries = new LinkedList<>();
/**
* offset of first log entry in the snippet

2
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java
View File

@ -193,7 +193,7 @@ public class DefaultCodeCommentManager extends AbstractEntityManager<CodeComment
command.after(DateUtils.addDays(oldestDate, -1));
command.revisions(Lists.newArrayList(commitId.name()));
command.count(MAX_HISTORY_COMMITS_TO_CHECK);
Set<String> revisions = new HashSet<>(command.call(null));
Set<String> revisions = new HashSet<>(command.call());
RevCommit commit = revWalk.parseCommit(commitId);
List<String> newLines = GitUtils.readLines(project.getRepository(), commit, path,

2
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
View File

@ -208,7 +208,7 @@ public class DefaultProjectManager extends AbstractEntityManager<Project> implem
public void fork(Project from, Project to) {
save(to);
FileUtils.cleanDir(to.getGitDir());
new CloneCommand(to.getGitDir()).mirror(true).from(from.getGitDir().getAbsolutePath()).call(null);
new CloneCommand(to.getGitDir()).mirror(true).from(from.getGitDir().getAbsolutePath()).call();
commitInfoManager.cloneInfo(from, to);
avatarManager.copyAvatar(from.getFacade(), to.getFacade());
}

8
server-core/src/main/java/io/onedev/server/git/GitFilter.java
View File

@ -157,7 +157,7 @@ public class GitFilter implements Filter {
try {
InputStream is = ServletUtils.getInputStream(request);
OutputStream os = response.getOutputStream();
new UploadCommand(gitDir, environments).input(is).output(os).call(null);
new UploadCommand(gitDir, environments).input(is).output(os).call();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -174,7 +174,7 @@ public class GitFilter implements Filter {
try {
InputStream is = ServletUtils.getInputStream(request);
OutputStream os = response.getOutputStream();
new ReceiveCommand(gitDir, environments).input(is).output(os).call(null);
new ReceiveCommand(gitDir, environments).input(is).output(os).call();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -221,13 +221,13 @@ public class GitFilter implements Filter {
if (service.contains("upload")) {
checkPullPermission(request, project);
writeInitial(response, service);
new AdvertiseUploadRefsCommand(gitDir).output(response.getOutputStream()).call(null);
new AdvertiseUploadRefsCommand(gitDir).output(response.getOutputStream()).call();
} else {
if (!SecurityUtils.canWriteCode(project.getFacade())) {
throw new UnauthorizedException("You do not have permission to push to this project.");
}
writeInitial(response, service);
new AdvertiseReceiveRefsCommand(gitDir).output(response.getOutputStream()).call(null);
new AdvertiseReceiveRefsCommand(gitDir).output(response.getOutputStream()).call();
}
}

6
server-core/src/main/java/io/onedev/server/git/GitUtils.java
View File

@ -327,7 +327,7 @@ public class GitUtils {
new FetchCommand(toRepository.getDirectory())
.from(fromRepository.getDirectory().getAbsolutePath())
.refspec(fetchRef)
.call(null);
.call();
} else {
LockUtils.call("repository-fetch:" + fromRepository.getDirectory(), new Callable<Void>() {
@ -340,7 +340,7 @@ public class GitUtils {
new FetchCommand(toRepository.getDirectory())
.from(fromRepository.getDirectory().getAbsolutePath())
.refspec(refUpdate.getName())
.call(null);
.call();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -356,7 +356,7 @@ public class GitUtils {
if (gitEnvs != null && !gitEnvs.isEmpty()) {
IsAncestorCommand cmd = new IsAncestorCommand(repository.getDirectory(), gitEnvs);
cmd.ancestor(base.name()).descendant(tip.name());
return cmd.call(null);
return cmd.call();
} else {
try (RevWalk revWalk = new RevWalk(repository)) {
RevCommit baseCommit;

8
server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java
View File

@ -27,20 +27,20 @@ public class AdvertiseReceiveRefsCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(output);
Logger effectiveLogger = logger!=null?logger:AdvertiseReceiveRefsCommand.logger;
Commandline cmd = cmd();
cmd.addArgs("receive-pack", "--stateless-rpc", "--advertise-refs", ".");
cmd.execute(output, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

8
server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java
View File

@ -27,20 +27,20 @@ public class AdvertiseUploadRefsCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(output);
Logger effectiveLogger = logger!=null?logger:AdvertiseUploadRefsCommand.logger;
Commandline cmd = cmd();
cmd.addArgs("upload-pack", "--stateless-rpc", "--advertise-refs", ".");
cmd.execute(output, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

4
server-core/src/main/java/io/onedev/server/git/command/BlameCommand.java
View File

@ -78,7 +78,7 @@ public class BlameCommand extends GitCommand<Collection<BlameBlock>> {
}
@Override
public Collection<BlameBlock> call(Logger logger) {
public Collection<BlameBlock> call() {
Preconditions.checkArgument(commitHash!=null && ObjectId.isId(commitHash), "commit hash has to be specified.");
Preconditions.checkNotNull(file, "file parameter has to be specified.");
@ -166,7 +166,7 @@ public class BlameCommand extends GitCommand<Collection<BlameBlock>> {
}
}
}, logger);
});
if (!endOfFile.get())
result.checkReturnCode();

9
server-core/src/main/java/io/onedev/server/git/command/CheckoutCommand.java
View File

@ -26,27 +26,26 @@ public class CheckoutCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(refspec, "refspec param has to be specified.");
Commandline cmd = cmd().addArgs("checkout", "--quiet", refspec);
Logger effectiveLogger = logger!=null?logger:CheckoutCommand.logger;
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.trace(line);
logger.trace(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

9
server-core/src/main/java/io/onedev/server/git/command/CleanCommand.java
View File

@ -24,28 +24,27 @@ public class CleanCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Commandline cmd = cmd().addArgs("clean");
if (options != null)
cmd.addArgs(options);
Logger effectiveLogger = logger!=null?logger:CleanCommand.logger;
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.trace(line);
logger.trace(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

13
server-core/src/main/java/io/onedev/server/git/command/CloneCommand.java
View File

@ -61,7 +61,7 @@ public class CloneCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(from, "from has to be specified.");
Commandline cmd = cmd().addArgs("clone");
@ -79,12 +79,11 @@ public class CloneCommand extends GitCommand<Void> {
cmd.addArgs(from);
cmd.addArgs(".");
Logger effectiveLogger = logger!=null?logger:CloneCommand.logger;
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.trace(line);
logger.trace(line);
}
}, new LineConsumer(){
@ -92,14 +91,14 @@ public class CloneCommand extends GitCommand<Void> {
@Override
public void consume(String line) {
if (line.startsWith("Cloning into ") || line.equals("done."))
effectiveLogger.trace(line);
logger.trace(line);
else if (line.contains("You appear to have cloned an empty repository"))
effectiveLogger.warn(line);
logger.warn(line);
else
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

9
server-core/src/main/java/io/onedev/server/git/command/FetchCommand.java
View File

@ -40,7 +40,7 @@ public class FetchCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(from, "from param has to be specified.");
Commandline cmd = cmd().addArgs("fetch");
@ -52,22 +52,21 @@ public class FetchCommand extends GitCommand<Void> {
for (String each: refspec)
cmd.addArgs(each);
Logger effectiveLogger = logger!=null?logger:FetchCommand.logger;
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.trace(line);
logger.trace(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

2
server-core/src/main/java/io/onedev/server/git/command/GitCommand.java
View File

@ -100,6 +100,6 @@ public abstract class GitCommand<V> {
return AppLoader.getInstance(GitConfig.class).getExecutable();
}
public abstract V call(@Nullable Logger logger);
public abstract V call();
}

11
server-core/src/main/java/io/onedev/server/git/command/InitCommand.java
View File

@ -33,7 +33,7 @@ public class InitCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(from, "from param has to be specified.");
Commandline cmd = cmd().addArgs("fetch");
@ -43,12 +43,11 @@ public class InitCommand extends GitCommand<Void> {
for (String each: refspec)
cmd.addArgs(each);
Logger effectiveLogger = logger!=null?logger:InitCommand.logger;
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.trace(line);
logger.trace(line);
}
}, new LineConsumer() {
@ -59,13 +58,13 @@ public class InitCommand extends GitCommand<Void> {
|| line.startsWith(" * branch")
|| line.startsWith(" * [new ref]")
|| line.contains("..") && line.contains("->")) {
effectiveLogger.info(line);
logger.info(line);
} else {
effectiveLogger.error(line);
logger.error(line);
}
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
}

7
server-core/src/main/java/io/onedev/server/git/command/IsAncestorCommand.java
View File

@ -35,7 +35,7 @@ public class IsAncestorCommand extends GitCommand<Boolean> {
}
@Override
public Boolean call(Logger logger) {
public Boolean call() {
Preconditions.checkNotNull(ancestor, "ancestor has to be specified.");
Preconditions.checkNotNull(descendant, "descendant has to be specified.");
@ -43,7 +43,6 @@ public class IsAncestorCommand extends GitCommand<Boolean> {
cmd.addArgs("merge-base", "--is-ancestor", ancestor, descendant);
Logger effectiveLogger = logger!=null?logger:IsAncestorCommand.logger;
ExecuteResult result = cmd.execute(new LineConsumer() {
@Override
@ -54,10 +53,10 @@ public class IsAncestorCommand extends GitCommand<Boolean> {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger);
});
if (result.getReturnCode() == 0)
return true;

7
server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java
View File

@ -45,7 +45,7 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
}
@Override
public Collection<String> call(Logger logger) {
public Collection<String> call() {
Preconditions.checkNotNull(toRev, "toRev has to be specified.");
Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");
@ -58,7 +58,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
if (path != null)
cmd.addArgs("--", path);
Logger effectiveLogger = logger!=null?logger:ListChangedFilesCommand.logger;
cmd.execute(new LineConsumer() {
@Override
@ -71,10 +70,10 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return changedFiles;
}

7
server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java
View File

@ -44,7 +44,7 @@ public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {
}
@Override
public Collection<FileChange> call(Logger logger) {
public Collection<FileChange> call() {
Preconditions.checkNotNull(toRev, "toRev has to be specified.");
Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");
@ -57,7 +57,6 @@ public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {
if (path != null)
cmd.addArgs("--", path);
Logger effectiveLogger = logger!=null?logger:ListFileChangesCommand.logger;
cmd.execute(new LineConsumer() {
@Override
@ -89,10 +88,10 @@ public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return fileChanges;
}

10
server-core/src/main/java/io/onedev/server/git/command/LogCommand.java
View File

@ -51,7 +51,7 @@ public abstract class LogCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkArgument(!revisions.isEmpty(), "Log revisions have to be specified");
Commandline cmd = cmd();
@ -84,8 +84,6 @@ public abstract class LogCommand extends GitCommand<Void> {
for (String revision: revisions)
cmd.addArgs(revision);
Logger effectiveLogger = logger!=null?logger:LogCommand.logger;
AtomicReference<GitCommit.Builder> commitBuilderRef = new AtomicReference<>(null);
cmd.execute(new LineConsumer() {
@ -159,13 +157,13 @@ public abstract class LogCommand extends GitCommand<Void> {
public void consume(String line) {
if (line.contains("inexact rename detection was skipped")
|| line.contains("you may want to set your diff.renameLimit variable")) {
effectiveLogger.trace(line);
logger.trace(line);
} else {
effectiveLogger.error(line);
logger.error(line);
}
}
}, logger).checkReturnCode();
}).checkReturnCode();
if (commitBuilderRef.get() != null)
consume(commitBuilderRef.get().build());

7
server-core/src/main/java/io/onedev/server/git/command/ReceiveCommand.java
View File

@ -36,22 +36,21 @@ public class ReceiveCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(input);
Preconditions.checkNotNull(output);
Commandline cmd = cmd();
cmd.addArgs("receive-pack", "--stateless-rpc", ".");
Logger effectiveLogger = logger!=null?logger:ReceiveCommand.logger;
cmd.execute(output, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, input, logger).checkReturnCode();
}, input).checkReturnCode();
return null;
}

8
server-core/src/main/java/io/onedev/server/git/command/RevListCommand.java
View File

@ -165,7 +165,7 @@ public class RevListCommand extends GitCommand<List<String>> {
}
@Override
public List<String> call(Logger logger) {
public List<String> call() {
Commandline cmd = cmd();
cmd.addArgs("rev-list");
@ -218,8 +218,6 @@ public class RevListCommand extends GitCommand<List<String>> {
for (String path: paths)
cmd.addArgs(path);
Logger effectiveLogger = logger!=null?logger:RevListCommand.logger;
List<String> commitHashes = new ArrayList<>();
cmd.execute(new LineConsumer() {
@ -232,10 +230,10 @@ public class RevListCommand extends GitCommand<List<String>> {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, logger).checkReturnCode();
}).checkReturnCode();
return commitHashes;
}

7
server-core/src/main/java/io/onedev/server/git/command/UploadCommand.java
View File

@ -36,22 +36,21 @@ public class UploadCommand extends GitCommand<Void> {
}
@Override
public Void call(Logger logger) {
public Void call() {
Preconditions.checkNotNull(input);
Preconditions.checkNotNull(output);
Commandline cmd = cmd();
cmd.addArgs("upload-pack", "--stateless-rpc", ".");
Logger effectiveLogger = logger!=null?logger:UploadCommand.logger;
cmd.execute(output, new LineConsumer() {
@Override
public void consume(String line) {
effectiveLogger.error(line);
logger.error(line);
}
}, input, logger).checkReturnCode();
}, input).checkReturnCode();
return null;
}

4
server-core/src/main/java/io/onedev/server/model/Project.java
View File

@ -1303,7 +1303,7 @@ public class Project extends AbstractEntity implements Validatable {
List<User> authors = new ArrayList<>();
UserManager userManager = OneDev.getInstance(UserManager.class);
for (BlameBlock block: cmd.call(null)) {
for (BlameBlock block: cmd.call()) {
User author = userManager.find(block.getCommit().getAuthor());
if (author != null && !authors.contains(author))
authors.add(author);
@ -1434,7 +1434,7 @@ public class Project extends AbstractEntity implements Validatable {
if (gitEnvs != null && !gitEnvs.isEmpty()) {
ListChangedFilesCommand cmd = new ListChangedFilesCommand(getGitDir(), gitEnvs);
cmd.fromRev(oldObjectId.name()).toRev(newObjectId.name());
return cmd.call(null);
return cmd.call();
} else {
return GitUtils.getChangedFiles(getRepository(), oldObjectId, newObjectId);
}

2
server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java
View File

@ -108,7 +108,7 @@ public class PullRequestUpdate extends AbstractEntity {
ListFileChangesCommand cmd = new ListFileChangesCommand(getRequest().getTargetProject().getGitDir());
cmd.fromRev(getBaseCommitHash());
cmd.toRev(getHeadCommitHash());
fileChanges = cmd.call(null);
fileChanges = cmd.call();
}
return fileChanges;
}

19
server-core/src/main/java/io/onedev/server/model/support/JobContext.java
View File

@ -9,10 +9,10 @@ import java.util.Map;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.ObjectId;
import org.slf4j.Logger;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.git.command.CheckoutCommand;
import io.onedev.server.git.command.FetchCommand;
import io.onedev.server.util.patternset.PatternSet;
@ -21,6 +21,8 @@ public abstract class JobContext {
private final String projectName;
private final File gitDir;
private final String environment;
private final File serverWorkspace;
@ -37,13 +39,14 @@ public abstract class JobContext {
private final PatternSet collectFiles;
private final Logger logger;
private final JobLogger logger;
public JobContext(String projectName, String environment, File workspace,
public JobContext(String projectName, File gitDir, String environment, File workspace,
Map<String, String> envVars, List<String> commands, boolean cloneSource,
ObjectId commitId, Collection<JobCache> caches, PatternSet collectFiles,
Logger logger) {
JobLogger logger) {
this.projectName = projectName;
this.gitDir = gitDir;
this.environment = environment;
this.serverWorkspace = workspace;
this.envVars = envVars;
@ -91,13 +94,13 @@ public abstract class JobContext {
return collectFiles;
}
public Logger getLogger() {
public JobLogger getLogger() {
return logger;
}
private void fetchAndCheckout(File gitDir) {
new FetchCommand(gitDir).depth(1).from(gitDir.getAbsolutePath()).refspec(commitId.name()).call(logger);
new CheckoutCommand(gitDir).refspec(commitId.name()).call(logger);
private void fetchAndCheckout(File checkoutDir) {
new FetchCommand(checkoutDir).depth(1).from(gitDir.getAbsolutePath()).refspec(commitId.name()).call();
new CheckoutCommand(checkoutDir).refspec(commitId.name()).call();
}
public void checkoutSource(File dir) {

2
server-core/src/main/java/io/onedev/server/model/support/JobExecutor.java
View File

@ -120,7 +120,7 @@ public abstract class JobExecutor implements Serializable {
this.cacheTTL = cacheTTL;
}
public abstract void execute(String jobId, JobContext context);
public abstract void execute(String jobToken, JobContext context);
public final boolean isApplicable(Project project, ObjectId commitId, String jobName, String environment) {
Matcher matcher = new ChildAwareMatcher();

10
server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java
View File

@ -15,8 +15,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import io.onedev.server.OneDev;
import io.onedev.server.ci.job.log.LogEntry;
import io.onedev.server.ci.job.log.LogManager;
import io.onedev.server.ci.job.log.JobLogEntry;
import io.onedev.server.ci.job.log.JobLogManager;
import io.onedev.server.ci.job.log.LogSnippet;
import io.onedev.server.model.Build;
import io.onedev.server.web.behavior.WebSocketObserver;
@ -39,7 +39,7 @@ public class BuildLogPanel extends GenericPanel<Build> {
add(new WebSocketObserver() {
private void appendRecentLogEntries(IPartialPageRequestHandler handler) {
List<LogEntry> logEntries = getLogManager().readLogEntries(getBuild(), nextOffset, 0);
List<JobLogEntry> logEntries = getLogManager().readLogEntries(getBuild(), nextOffset, 0);
if (!logEntries.isEmpty()) {
nextOffset += logEntries.size();
@ -70,8 +70,8 @@ public class BuildLogPanel extends GenericPanel<Build> {
setOutputMarkupId(true);
}
private LogManager getLogManager() {
return OneDev.getInstance(LogManager.class);
private JobLogManager getLogManager() {
return OneDev.getInstance(JobLogManager.class);
}
private String asJSON(Object obj) {

19
server-core/src/main/java/io/onedev/server/web/component/build/log/build-log.css
View File

@ -4,26 +4,11 @@
color: white;
position: relative;
}
.build-log>.log-entry.ERROR {
color: red;
}
.build-log>.log-entry.WARN {
color: yellow;
}
.build-log>.log-entry.DEBUG {
color: lightgray;
}
.build-log>.log-entry.TRACE {
color: darkgray;
}
.build-log>.log-entry>* {
.build-log>.log-entry>.date {
padding-right: 8px;
}
.build-log>.log-entry>.message {
padding-right: 0;
}
.build-log>.too-many-entries, .build-log>.no-entries {
color: red;
color: yellow;
font-size: 16px;
font-weight: bold;
}

3
server-core/src/main/java/io/onedev/server/web/component/build/log/build-log.js
View File

@ -8,9 +8,8 @@ onedev.server.buildLog = {
onedev.server.buildLog.appendLogEntries(containerId, logEntries, maxNumOfLogEntries);
},
renderLogEntry: function(logEntry) {
var $logEntry = $("<div class='log-entry " + logEntry.level + "'></div>");
var $logEntry = $("<div class='log-entry'></div>");
$logEntry.append("<span class='date'>" + moment(logEntry.date).format("HH:mm:ss") + "</span>");
$logEntry.append("<span class='log-level'>" + logEntry.level + "</span>");
var $message = $("<span class='message'></span>");
$message.text(logEntry.message);
$logEntry.append($message);

2
server-core/src/main/java/io/onedev/server/web/component/commit/list/CommitListPanel.java
View File

@ -160,7 +160,7 @@ public class CommitListPanel extends Panel {
if (command.revisions().isEmpty() && getCompareWith() != null)
command.revisions(Lists.newArrayList(getCompareWith()));
commitHashes = command.call(null);
commitHashes = command.call();
} catch (Exception e) {
if (e.getMessage() != null)
error(e.getMessage());

4
server-core/src/main/java/io/onedev/server/web/component/diff/blob/text/TextDiffPanel.java
View File

@ -155,7 +155,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
String oldPath = change.getOldBlobIdent().path;
if (oldPath != null) {
cmd.commitHash(getOldCommit().name()).file(oldPath);
for (BlameBlock blame: cmd.call(null)) {
for (BlameBlock blame: cmd.call()) {
for (LinearRange range: blame.getRanges()) {
for (int i=range.getFrom(); i<=range.getTo(); i++)
blameInfo.oldBlame.put(i, blame.getCommit());
@ -165,7 +165,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
String newPath = change.getNewBlobIdent().path;
if (newPath != null) {
cmd.commitHash(getNewCommit().name()).file(newPath);
for (BlameBlock blame: cmd.call(null)) {
for (BlameBlock blame: cmd.call()) {
for (LinearRange range: blame.getRanges()) {
for (int i=range.getFrom(); i<=range.getTo(); i++)
blameInfo.newBlame.put(i, blame.getCommit());

4
server-core/src/main/java/io/onedev/server/web/download/BuildLogDownloadResource.java
View File

@ -17,7 +17,7 @@ import org.apache.wicket.request.resource.AbstractResource;
import com.google.common.base.Charsets;
import io.onedev.server.OneDev;
import io.onedev.server.ci.job.log.LogManager;
import io.onedev.server.ci.job.log.JobLogManager;
import io.onedev.server.entitymanager.BuildManager;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Build;
@ -75,7 +75,7 @@ public class BuildLogDownloadResource extends AbstractResource {
@Override
public void writeData(Attributes attributes) throws IOException {
try (InputStream is = OneDev.getInstance(LogManager.class).openLogStream(build)) {
try (InputStream is = OneDev.getInstance(JobLogManager.class).openLogStream(build)) {
IOUtils.copy(is, attributes.getResponse().getOutputStream());
}
}

3
server-core/src/main/java/io/onedev/server/web/page/base/base.js
View File

@ -279,6 +279,9 @@ onedev.server = {
focusOn: function(componentId) {
if (componentId)
onedev.server.focus.doFocus($("#" + componentId));
else if (document.activeElement != document.body)
document.activeElement.blur();
onedev.server.focus.$components = null;
},

2
server-core/src/main/java/io/onedev/server/web/page/project/blob/render/renderers/source/SourceViewPanel.java
View File

@ -839,7 +839,7 @@ public class SourceViewPanel extends BlobViewPanel implements Positionable, Sear
BlameCommand cmd = new BlameCommand(context.getProject().getGitDir());
cmd.commitHash(commitHash).file(context.getBlobIdent().path);
for (BlameBlock blame: cmd.call(null)) {
for (BlameBlock blame: cmd.call()) {
BlameInfo blameInfo = new BlameInfo();
blameInfo.commitDate = DateUtils.formatDate(blame.getCommit().getCommitter().getWhen());
blameInfo.authorName = HtmlEscape.escapeHtml5(blame.getCommit().getAuthor().getName());

1
server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java
View File

@ -232,6 +232,7 @@ public abstract class BuildDetailPage extends ProjectPage implements InputContex
};
}
target.focusComponent(null);
}
@Override

1
server-core/src/main/java/io/onedev/server/web/page/test/TestPage.html
View File

@ -1,2 +1,3 @@
<wicket:extend>
<a wicket:id="test">test</a>
</wicket:extend>

9
server-core/src/main/java/io/onedev/server/web/page/test/TestPage.java
View File

@ -3,6 +3,7 @@ package io.onedev.server.web.page.test;
import org.apache.wicket.markup.head.IHeaderResponse;
import org.apache.wicket.markup.head.JavaScriptHeaderItem;
import org.apache.wicket.markup.head.OnDomReadyHeaderItem;
import org.apache.wicket.markup.html.link.Link;
import org.apache.wicket.request.mapper.parameter.PageParameters;
import io.onedev.server.web.page.base.BasePage;
@ -17,6 +18,14 @@ public class TestPage extends BasePage {
@Override
protected void onInitialize() {
super.onInitialize();
add(new Link<Void>("test") {
@Override
public void onClick() {
}
});
}
@Override

6
server-core/src/test/java/io/onedev/server/git/command/BlameCommandTest.java
View File

@ -31,7 +31,7 @@ public class BlameCommandTest extends AbstractGitTest {
Collection<BlameBlock> blames = new BlameCommand(git.getRepository().getDirectory())
.commitHash(commitHash)
.file("file")
.call(null);
.call();
assertEquals(1, blames.size());
assertEquals(commitHash + ": 0-8", blames.iterator().next().toString());
@ -53,7 +53,7 @@ public class BlameCommandTest extends AbstractGitTest {
.commitHash(commitHash)
.file("file")
.range(new LinearRange(5, 8))
.call(null);
.call();
assertEquals(2, blames.size());
assertEquals(commitHash + ": 8-8", getBlock(blames, commitHash).toString());
@ -69,7 +69,7 @@ public class BlameCommandTest extends AbstractGitTest {
blames = new BlameCommand(git.getRepository().getDirectory())
.commitHash(commitHash)
.file("file")
.call(null);
.call();
commitHash = git.getRepository().resolve("master~1").name();

2
server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java
View File

@ -78,7 +78,7 @@ public class LogCommandTest extends AbstractGitTest {
commits.add(commit);
}
}.revisions(Lists.newArrayList("master")).call(null);
}.revisions(Lists.newArrayList("master")).call();
assertEquals(2, commits.size());

4
server-plugin/server-plugin-artifact/src/main/java/io/onedev/server/plugin/artifact/ArtifactsPopulator.java
View File

@ -4,17 +4,17 @@ import java.io.File;
import java.util.concurrent.Callable;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import io.onedev.commons.utils.LockUtils;
import io.onedev.server.ci.job.DependencyPopulator;
import io.onedev.server.ci.job.JobOutcome;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.Build;
public class ArtifactsPopulator implements DependencyPopulator {
@Override
public void populate(Build dependency, File workspace, Logger logger) {
public void populate(Build dependency, File workspace, JobLogger logger) {
File outcomeDir = JobOutcome.getOutcomeDir(dependency, JobArtifacts.DIR);
LockUtils.read(JobOutcome.getLockKey(dependency, JobArtifacts.DIR), new Callable<Void>() {

5
server-plugin/server-plugin-artifact/src/main/java/io/onedev/server/plugin/artifact/JobArtifacts.java
View File

@ -4,11 +4,10 @@ import java.io.File;
import java.io.IOException;
import java.util.concurrent.Callable;
import org.slf4j.Logger;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.LockUtils;
import io.onedev.server.ci.job.JobOutcome;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.Build;
import io.onedev.server.web.editable.annotation.Editable;
@ -20,7 +19,7 @@ public class JobArtifacts extends JobOutcome {
public static final String DIR = "artifacts";
@Override
public void process(Build build, File workspace, Logger logger) {
public void process(Build build, File workspace, JobLogger logger) {
File outcomeDir = getOutcomeDir(build, DIR);
FileUtils.createDir(outcomeDir);

6
server-plugin/server-plugin-htmlreport/src/main/java/io/onedev/server/plugin/htmlreport/JobHtmlReport.java
View File

@ -8,11 +8,11 @@ import java.util.concurrent.Callable;
import org.apache.commons.lang3.SerializationUtils;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.LockUtils;
import io.onedev.server.ci.job.JobOutcome;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.Build;
import io.onedev.server.web.editable.annotation.Editable;
@ -50,7 +50,7 @@ public class JobHtmlReport extends JobOutcome {
}
@Override
public void process(Build build, File workspace, Logger logger) {
public void process(Build build, File workspace, JobLogger logger) {
File outcomeDir = getOutcomeDir(build, DIR);
FileUtils.createDir(outcomeDir);
@ -78,7 +78,7 @@ public class JobHtmlReport extends JobOutcome {
}
}
} else {
logger.warn("Html report start page not found: " + startPage.getAbsolutePath());
logger.log("ERROR: Html report start page not found: " + startPage.getAbsolutePath());
}
return null;
}

588
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
View File

@ -3,22 +3,25 @@ package io.onedev.server.plugin.kubernetes;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import org.apache.commons.codec.Charsets;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
@ -26,11 +29,13 @@ import com.google.common.collect.Lists;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.Maps;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.ExecuteResult;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.server.OneDev;
import io.onedev.server.OneException;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.entitymanager.SettingManager;
import io.onedev.server.model.support.JobContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.plugin.kubernetes.KubernetesExecutor.TestData;
@ -42,7 +47,7 @@ import io.onedev.server.web.util.Testable;
public class KubernetesExecutor extends JobExecutor implements Testable<TestData> {
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(KubernetesExecutor.class);
private String configFile;
@ -125,7 +130,20 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
}
@Override
public void execute(String jobId, JobContext context) {
public void execute(String jobToken, JobContext jobContext) {
execute(jobContext.getEnvironment(), jobToken, jobContext.getLogger(), jobContext);
}
@Override
public void test(TestData testData) {
execute(testData.getDockerImage(), KubernetesResource.TEST_JOB_TOKEN, new JobLogger() {
@Override
public void log(String message, Throwable t) {
logger.info(message, t);
}
}, null);
}
@Override
@ -147,28 +165,30 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
return cmdline;
}
private String createResource(Map<Object, Object> resourceData, Logger logger) {
private String createResource(Map<Object, Object> resourceDef, JobLogger logger) {
Commandline kubectl = newKubeCtl();
File file = null;
try {
AtomicReference<String> resourceNameRef = new AtomicReference<String>(null);
file = File.createTempFile("k8s", ".yaml");
FileUtils.writeFile(file, new Yaml().dump(resourceData), Charsets.UTF_8.name());
kubectl.addArgs("create", "-f", file.getAbsolutePath());
String resourceYaml = new Yaml().dump(resourceDef);
KubernetesExecutor.logger.trace("Kubernetes: creating resource with yaml:\n" + resourceYaml);
FileUtils.writeFile(file, resourceYaml, Charsets.UTF_8.name());
kubectl.addArgs("create", "-f", file.getAbsolutePath(), "-o", "jsonpath={.metadata.name}");
kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.info(line);
line = StringUtils.substringAfter(line, "/");
resourceNameRef.set(StringUtils.substringBefore(line, " "));
resourceNameRef.set(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
}).checkReturnCode();
@ -182,45 +202,44 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
}
}
private void deleteResource(String resourceType, String resourceName, Logger logger) {
private void deleteResource(String resourceType, String resourceName, JobLogger logger) {
Commandline cmd = newKubeCtl();
cmd.addArgs("delete", resourceType, resourceName, "--namespace=" + getNamespace());
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.info(line);
KubernetesExecutor.logger.debug(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
}).checkReturnCode();
}
private void createNamespaceIfNotExist(Logger logger) {
private void createNamespaceIfNotExist(JobLogger logger) {
Commandline cmd = newKubeCtl();
cmd.addArgs("get", "namespaces");
String query = String.format("{.items[?(@.metadata.name=='%s')]}", getNamespace());
cmd.addArgs("get", "namespaces", "-o", "jsonpath=" + query);
AtomicBoolean hasNamespace = new AtomicBoolean(false);
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.debug(line);
if (line.startsWith(getNamespace() + " "))
hasNamespace.set(true);
hasNamespace.set(true);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
}).checkReturnCode();
@ -232,28 +251,20 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
@Override
public void consume(String line) {
logger.debug(line);
KubernetesExecutor.logger.debug(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
}).checkReturnCode();
}
}
private String getResourceNamePrefix() {
try {
return "onedev-ci-" + InetAddress.getLocalHost().getHostName() + "-" + getName() + "-";
} catch (UnknownHostException e) {
throw new RuntimeException(e);
}
}
private List<Object> getImagePullSecretsData() {
List<Object> data = new ArrayList<>();
if (getImagePullSecrets() != null) {
@ -270,168 +281,424 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
return data;
}
private String getOS(Logger logger) {
logger.info("Checking OS...");
private String getOSName(JobLogger logger) {
logger.log("Checking working node OS...");
Commandline kubectl = newKubeCtl();
kubectl.addArgs("get", "nodes", "-o=jsonpath={..nodeInfo.operatingSystem}");
kubectl.addArgs("get", "nodes", "-o", "jsonpath={..nodeInfo.operatingSystem}");
for (NodeSelectorEntry entry: getNodeSelector())
kubectl.addArgs("-l", entry.getLabelName() + "=" + entry.getLabelValue());
AtomicReference<String> osRef = new AtomicReference<>(null);
AtomicReference<String> osNameRef = new AtomicReference<>(null);
kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
osRef.set(line);
osNameRef.set(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
}).checkReturnCode();
return Preconditions.checkNotNull(osRef.get(), "No applicable working nodes for this executor");
String osName = osNameRef.get();
if (osName != null) {
logger.log(String.format("OS of working node is '%s'", osName));
return osName;
} else {
throw new OneException("No applicable working nodes found for executor '" + getName() + "'");
}
}
@Override
public void test(TestData testData) {
private String getServerUrl() {
return OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
}
private List<Map<Object, Object>> getSecretEnvs(String secretName, Collection<String> secretKeys) {
List<Map<Object, Object>> secretEnvs = new ArrayList<>();
for (String secretKey: secretKeys) {
Map<Object, Object> secretEnv = new LinkedHashMap<>();
secretEnv.put("name", secretKey);
secretEnv.put("valueFrom", Maps.newLinkedHashMap("secretKeyRef", Maps.newLinkedHashMap(
"name", secretName,
"key", secretKey)));
secretEnvs.add(secretEnv);
}
return secretEnvs;
}
private void execute(String dockerImage, String jobToken, JobLogger logger, @Nullable JobContext jobContext) {
createNamespaceIfNotExist(logger);
String os = getOS(logger);
Map<String, Object> podSpec = new LinkedHashMap<>();
Map<Object, Object> containerSpec = Maps.newHashMap(
"name", "test",
"image", testData.getDockerImage());
if (os.equalsIgnoreCase("linux")) {
containerSpec.put("command", Lists.newArrayList("sh"));
containerSpec.put("args", Lists.newArrayList("-c", "echo hello from container"));
} else {
containerSpec.put("command", Lists.newArrayList("cmd"));
containerSpec.put("args", Lists.newArrayList("/c", "echo hello from container"));
}
podSpec.put("containers", Lists.<Object>newArrayList(containerSpec));
Map<String, String> nodeSelectorData = getNodeSelectorData();
if (!nodeSelectorData.isEmpty())
podSpec.put("nodeSelector", nodeSelectorData);
List<Object> imagePullSecretsData = getImagePullSecretsData();
if (!imagePullSecretsData.isEmpty())
podSpec.put("imagePullSecrets", imagePullSecretsData);
if (getServiceAccount() != null)
podSpec.put("serviceAccountName", getServiceAccount());
podSpec.put("restartPolicy", "Never");
Map<Object, Object> podData = Maps.newLinkedHashMap(
"apiVersion", "v1",
"kind", "Pod",
"metadata", Maps.newLinkedHashMap(
"generateName", getResourceNamePrefix() + "test-",
"namespace", getNamespace()),
"spec", podSpec);
String podName = createResource(podData, logger);
Map<String, String> secrets = Maps.newLinkedHashMap(KubernetesHelper.ENV_JOB_TOKEN, jobToken);
String secretName = createSecret(secrets, logger);
try {
waitForPod(podName, logger);
String osName = getOSName(logger);
Map<String, Object> podSpec = new LinkedHashMap<>();
Map<Object, Object> mainContainerSpec = Maps.newHashMap(
"name", "main",
"image", dockerImage);
Map<String, String> emptyDirMount = new LinkedHashMap<>();
String classPath;
if (osName.equalsIgnoreCase("linux")) {
mainContainerSpec.put("command", Lists.newArrayList("sh"));
mainContainerSpec.put("args", Lists.newArrayList(".onedev/job-commands-wrapper.sh"));
emptyDirMount.put("mountPath", "/onedev-workspace");
classPath = "/k8s-helper/*";
} else {
mainContainerSpec.put("command", Lists.newArrayList("cmd"));
mainContainerSpec.put("args", Lists.newArrayList("/c", ".onedev\\job-commands-wrapper.bat"));
emptyDirMount.put("mountPath", "C:\\onedev-workspace");
classPath = "C:\\k8s-helper\\*";
}
mainContainerSpec.put("workingDir", emptyDirMount.get("mountPath"));
emptyDirMount.put("name", "workspace");
mainContainerSpec.put("volumeMounts", Lists.<Object>newArrayList(emptyDirMount));
Map<Object, Object> resources = Maps.newLinkedHashMap("requests", Maps.newLinkedHashMap("cpu", "1"));
mainContainerSpec.put("resources", resources);
List<Map<Object, Object>> envs = new ArrayList<>();
Map<Object, Object> serverUrlEnv = Maps.newLinkedHashMap(
"name", KubernetesHelper.ENV_SERVER_URL,
"value", getServerUrl());
envs.add(serverUrlEnv);
envs.addAll(getSecretEnvs(secretName, secrets.keySet()));
List<String> sidecarArgs = Lists.newArrayList("-classpath", classPath, "io.onedev.k8shelper.SideCar");
List<String> initArgs = Lists.newArrayList("-classpath", classPath, "io.onedev.k8shelper.Init");
if (jobContext == null) {
sidecarArgs.add("test");
initArgs.add("test");
}
Map<Object, Object> sidecarContainerSpec = Maps.newHashMap(
"name", "sidecar",
"image", "1dev/k8s-helper",
"command", Lists.newArrayList("java"),
"args", sidecarArgs,
"env", envs,
"volumeMounts", Lists.<Object>newArrayList(emptyDirMount));
Map<Object, Object> initContainerSpec = Maps.newHashMap(
"name", "init",
"image", "1dev/k8s-helper",
"command", Lists.newArrayList("java"),
"args", initArgs,
"env", envs,
"resources", resources,
"volumeMounts", Lists.<Object>newArrayList(emptyDirMount));
podSpec.put("containers", Lists.<Object>newArrayList(mainContainerSpec, sidecarContainerSpec));
podSpec.put("initContainers", Lists.<Object>newArrayList(initContainerSpec));
Map<String, String> nodeSelectorData = getNodeSelectorData();
if (!nodeSelectorData.isEmpty())
podSpec.put("nodeSelector", nodeSelectorData);
List<Object> imagePullSecretsData = getImagePullSecretsData();
if (!imagePullSecretsData.isEmpty())
podSpec.put("imagePullSecrets", imagePullSecretsData);
if (getServiceAccount() != null)
podSpec.put("serviceAccountName", getServiceAccount());
podSpec.put("restartPolicy", "Never");
podSpec.put("volumes", Lists.<Object>newArrayList(Maps.newLinkedHashMap(
"name", "workspace",
"emptyDir", Maps.newLinkedHashMap())));
Map<Object, Object> podDef = Maps.newLinkedHashMap(
"apiVersion", "v1",
"kind", "Pod",
"metadata", Maps.newLinkedHashMap(
"generateName", "job-",
"namespace", getNamespace()),
"spec", podSpec);
String podName = createResource(podDef, logger);
try {
logger.log("Preparing job environment...");
watchPod(podName, new StatusChecker() {
@Override
public StopWatch check(JsonNode statusNode) {
JsonNode initContainerStatusesNode = statusNode.get("initContainerStatuses");
if (initContainerStatusesNode != null) {
for (JsonNode initContainerStatusNode: initContainerStatusesNode) {
JsonNode stateNode = initContainerStatusNode.get("state");
if (initContainerStatusNode.get("name").asText().equals("init")
&& (stateNode.get("running") != null || stateNode.get("terminated") != null)) {
return new StopWatch(null);
}
}
}
return null;
}
}, logger);
if (jobContext != null)
jobContext.notifyJobRunning();
waitForContainerStop(podName, "init", logger);
watchPod(podName, new StatusChecker() {
@Override
public StopWatch check(JsonNode statusNode) {
JsonNode initContainerStatusesNode = statusNode.get("initContainerStatuses");
String errorMessage = getContainerError(initContainerStatusesNode, "init");
if (errorMessage != null)
return new StopWatch(new OneException("Error executing init logic: " + errorMessage));
JsonNode containerStatusesNode = statusNode.get("containerStatuses");
if (isContainerStarted(containerStatusesNode, "main"))
return new StopWatch(null);
else
return null;
}
}, logger);
waitForContainerStop(podName, "main", logger);
watchPod(podName, new StatusChecker() {
@Override
public StopWatch check(JsonNode statusNode) {
JsonNode containerStatusesNode = statusNode.get("containerStatuses");
String errorMessage = getContainerError(containerStatusesNode, "main");
if (errorMessage != null)
return new StopWatch(new OneException(errorMessage));
if (isContainerStarted(containerStatusesNode, "sidecar"))
return new StopWatch(null);
else
return null;
}
}, logger);
waitForContainerStop(podName, "sidecar", logger);
watchPod(podName, new StatusChecker() {
@Override
public StopWatch check(JsonNode statusNode) {
JsonNode containerStatusesNode = statusNode.get("containerStatuses");
String errorMessage = getContainerError(containerStatusesNode, "sidecar");
if (errorMessage != null)
return new StopWatch(new OneException("Error executing sidecar logic: " + errorMessage));
else if (isContainerStopped(containerStatusesNode, "sidecar"))
return new StopWatch(null);
else
return null;
}
}, logger);
} finally {
deleteResource("pod", podName, logger);
}
} finally {
deleteResource("pod", podName, logger);
deleteResource("secret", secretName, logger);
}
}
private void waitForPod(String podName, Logger logger) {
Thread thread = Thread.currentThread();
AtomicBoolean podStartedRef = new AtomicBoolean(false);
AtomicReference<String> podErrorRef = new AtomicReference<String>(null);
@Nullable
private String getContainerError(@Nullable JsonNode containerStatusesNode, String containerName) {
if (containerStatusesNode != null) {
for (JsonNode containerStatusNode: containerStatusesNode) {
JsonNode stateNode = containerStatusNode.get("state");
if (containerStatusNode.get("name").asText().equals(containerName)) {
JsonNode terminatedNode = stateNode.get("terminated");
if (terminatedNode != null) {
String reason = terminatedNode.get("reason").asText();
if (!reason.equals("Completed")) {
JsonNode messageNode = terminatedNode.get("message");
if (messageNode != null) {
return messageNode.asText();
} else {
JsonNode exitCodeNode = terminatedNode.get("exitCode");
if (exitCodeNode != null && exitCodeNode.asInt() != 0)
return "exit code: " + exitCodeNode.asText();
else
return reason;
}
}
}
break;
}
}
}
return null;
}
private boolean isContainerStarted(@Nullable JsonNode containerStatusesNode, String containerName) {
if (containerStatusesNode != null) {
for (JsonNode containerStatusNode: containerStatusesNode) {
if (containerStatusNode.get("name").asText().equals(containerName)) {
JsonNode stateNode = containerStatusNode.get("state");
if (stateNode.get("running") != null || stateNode.get("terminated") != null)
return true;
break;
}
}
}
return false;
}
private boolean isContainerStopped(@Nullable JsonNode containerStatusesNode, String containerName) {
if (containerStatusesNode != null) {
for (JsonNode containerStatusNode: containerStatusesNode) {
if (containerStatusNode.get("name").asText().equals(containerName)) {
JsonNode stateNode = containerStatusNode.get("state");
if (stateNode.get("terminated") != null)
return true;
break;
}
}
}
return false;
}
private String createSecret(Map<String, String> secrets, JobLogger logger) {
Map<String, String> encodedSecrets = new LinkedHashMap<>();
for (Map.Entry<String, String> entry: secrets.entrySet())
encodedSecrets.put(entry.getKey(), Base64.getEncoder().encodeToString(entry.getValue().getBytes(Charsets.UTF_8)));
Map<Object, Object> secretDef = Maps.newLinkedHashMap(
"apiVersion", "v1",
"kind", "Secret",
"metadata", Maps.newLinkedHashMap(
"generateName", "secret-",
"namespace", getNamespace()),
"data", encodedSecrets);
return createResource(secretDef, logger);
}
private void watchPod(String podName, StatusChecker statusChecker, JobLogger logger) {
Commandline kubectl = newKubeCtl();
kubectl.addArgs("get", "event", "-n", getNamespace(), "--no-headers",
"--field-selector", "involvedObject.name=" + podName, "--watch");
ObjectMapper mapper = new ObjectMapper();
AtomicReference<StopWatch> stopWatchRef = new AtomicReference<>(null);
StringBuilder json = new StringBuilder();
kubectl.addArgs("get", "pod", podName, "-n", getNamespace(), "--watch", "-o", "json");
Thread thread = Thread.currentThread();
try {
kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
StringTokenizer tokenizer = new StringTokenizer(line);
tokenizer.nextToken();
String type = tokenizer.nextToken();
tokenizer.nextToken();
tokenizer.nextToken();
String message = tokenizer.nextToken("\n").trim();
if (type.equals("Normal"))
logger.info(message);
else
logger.error(message);
if (!type.equals("Normal") && !message.contains("Insufficient cpu")) {
podErrorRef.set(message);
thread.interrupt();
} else if (message.startsWith("Started container")) {
podStartedRef.set(true);
thread.interrupt();
if (line.startsWith("{")) {
json.append("{").append("\n");
} else if (line.startsWith("}")) {
json.append("}");
try {
process(mapper.readTree(json.toString()));
} catch (IOException e) {
KubernetesExecutor.logger.error("Error reading json", e);
}
json.setLength(0);
} else {
json.append(line).append("\n");
}
}
private void process(JsonNode podNode) {
String errorMessage = null;
JsonNode statusNode = podNode.get("status");
JsonNode conditionsNode = statusNode.get("conditions");
if (conditionsNode != null) {
for (JsonNode conditionNode: conditionsNode) {
if (conditionNode.get("type").asText().equals("PodScheduled")
&& conditionNode.get("status").asText().equals("False")
&& conditionNode.get("reason").asText().equals("Unschedulable")) {
logger.log(conditionNode.get("message").asText());
}
}
}
Collection<JsonNode> containerStatusNodes = new ArrayList<>();
JsonNode initContainerStatusesNode = statusNode.get("initContainerStatuses");
if (initContainerStatusesNode != null) {
for (JsonNode containerStatusNode: initContainerStatusesNode)
containerStatusNodes.add(containerStatusNode);
}
JsonNode containerStatusesNode = statusNode.get("containerStatuses");
if (containerStatusesNode != null) {
for (JsonNode containerStatusNode: containerStatusesNode)
containerStatusNodes.add(containerStatusNode);
}
for (JsonNode containerStatusNode: containerStatusNodes) {
JsonNode stateNode = containerStatusNode.get("state");
JsonNode waitingNode = stateNode.get("waiting");
if (waitingNode != null) {
String reason = waitingNode.get("reason").asText();
if (reason.equals("ErrImagePull") || reason.equals("InvalidImageName")
|| reason.equals("ImageInspectError") || reason.equals("ErrImageNeverPull")
|| reason.equals("RegistryUnavailable")) {
JsonNode messageNode = waitingNode.get("message");
if (messageNode != null)
errorMessage = messageNode.asText();
else
errorMessage = reason;
break;
}
}
}
if (errorMessage != null)
stopWatchRef.set(new StopWatch(new OneException(errorMessage)));
else
stopWatchRef.set(statusChecker.check(statusNode));
if (stopWatchRef.get() != null)
thread.interrupt();
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
logger.log("Kubernetes: " + line);
}
});
}).checkReturnCode();
throw new OneException("Unexpected end of pod event watching");
throw new OneException("Unexpected end of pod watching");
} catch (Exception e) {
if (ExceptionUtils.find(e, InterruptedException.class) != null) {
if (podStartedRef.get()) {
kubectl = newKubeCtl();
kubectl.addArgs("logs", podName, "-n", getNamespace(), "--follow");
while (true) {
AtomicReference<Boolean> containerCreatingRef = new AtomicReference<Boolean>(false);
ExecuteResult result = kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.info(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
if (line.contains("is waiting to start: ContainerCreating"))
containerCreatingRef.set(true);
else
logger.error(line);
}
});
if (containerCreatingRef.get()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e2) {
throw new RuntimeException(e2);
}
} else {
result.checkReturnCode();
break;
}
}
} else if (podErrorRef.get() != null) {
throw new OneException(podErrorRef.get());
} else {
throw e;
}
} else {
throw e;
StopWatch stopWatch = stopWatchRef.get();
if (stopWatch != null) {
if (stopWatch.getException() != null)
throw stopWatch.getException();
} else {
throw ExceptionUtils.unchecked(e);
}
}
}
}
private void waitForContainerStop(String podName, String containerName, JobLogger logger) {
Commandline kubectl = newKubeCtl();
kubectl.addArgs("logs", podName, "-c", containerName, "-n", getNamespace(), "--follow");
kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.log(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.log(line);
}
}).checkReturnCode();
}
@Editable
@ -463,6 +730,27 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
}
private static interface StatusChecker {
StopWatch check(JsonNode statusNode);
}
private static class StopWatch {
private final RuntimeException exception;
public StopWatch(@Nullable RuntimeException exception) {
this.exception = exception;
}
@Nullable
public RuntimeException getException() {
return exception;
}
}
@Editable(name="Specify a Docker Image to Test Against")
public static class TestData implements Serializable {

113
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesHelperTester.java
View File

@ -1,113 +0,0 @@
package io.onedev.server.plugin.kubernetes;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.apache.commons.lang.SystemUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.SettingManager;
import io.onedev.server.model.support.JobContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.web.editable.annotation.Editable;
@Editable(order=400)
public class KubernetesHelperTester extends JobExecutor {
private static final long serialVersionUID = 1L;
@Override
public void execute(String jobId, JobContext context) {
context.notifyJobRunning();
SettingManager settingManager = OneDev.getInstance(SettingManager.class);
String serverUrl = settingManager.getSystemSetting().getServerUrl();
File workspace = FileUtils.createTempDir("k8s-workspace");
try {
KubernetesHelper.init(serverUrl, jobId, workspace);
Future<?> sidecar = OneDev.getInstance(ExecutorService.class).submit(new Runnable() {
@Override
public void run() {
KubernetesHelper.sidecar(serverUrl, jobId, workspace);
}
});
try {
Commandline cmd;
if (SystemUtils.IS_OS_WINDOWS) {
File scriptFile = new File(workspace, "onedev-job-commands.bat");
try {
FileUtils.writeLines(scriptFile, context.getCommands(), "\r\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
cmd = new Commandline("cmd");
cmd.addArgs("/c", scriptFile.getAbsolutePath());
} else {
File scriptFile = new File(workspace, "onedev-job-commands.sh");
try {
FileUtils.writeLines(scriptFile, context.getCommands(), "\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
cmd = new Commandline("sh");
cmd.addArgs(scriptFile.getAbsolutePath());
}
cmd.workingDir(workspace);
cmd.environments(context.getEnvVars());
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
context.getLogger().info(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
context.getLogger().error(line);
}
}).checkReturnCode();
} finally {
try {
new File(workspace, KubernetesHelper.JOB_FINISH_FILE).createNewFile();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
try {
sidecar.get();
} catch (InterruptedException e) {
sidecar.cancel(true);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
} finally {
FileUtils.deleteDir(workspace);
}
}
@Override
public void checkCaches() {
}
@Override
public void cleanDir(File dir) {
FileUtils.cleanDir(dir);
}
}

6
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesModule.java
View File

@ -6,7 +6,6 @@ import org.glassfish.jersey.server.ResourceConfig;
import com.google.common.collect.Sets;
import io.onedev.commons.launcher.bootstrap.Bootstrap;
import io.onedev.commons.launcher.loader.AbstractPluginModule;
import io.onedev.commons.launcher.loader.ImplementationProvider;
import io.onedev.server.model.support.JobExecutor;
@ -32,10 +31,7 @@ public class KubernetesModule extends AbstractPluginModule {
@Override
public Collection<Class<?>> getImplementations() {
Collection<Class<?>> implementations = Sets.newHashSet(KubernetesExecutor.class);
if (Bootstrap.sandboxMode)
implementations.add(KubernetesHelperTester.class);
return implementations;
return Sets.newHashSet(KubernetesExecutor.class);
}
});

23
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesResource.java
View File

@ -6,6 +6,7 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -32,6 +33,8 @@ import io.onedev.server.model.support.JobContext;
@Singleton
public class KubernetesResource {
public static final String TEST_JOB_TOKEN = UUID.randomUUID().toString();
private final JobManager jobManager;
@Context
@ -84,13 +87,23 @@ public class KubernetesResource {
return Response.ok().build();
}
@GET
@Path("/test")
public Response test() {
String jobToken = request.getHeader(JobManager.JOB_TOKEN_HTTP_HEADER);
if (TEST_JOB_TOKEN.equals(jobToken))
return Response.ok().build();
else
return Response.status(400).entity("Invalid or no job token").build();
}
private JobContext getJobContext() {
String jobId = request.getHeader(JobManager.JOB_ID_HTTP_HEADER);
if (jobId == null)
throw new OneException("Http header '" + JobManager.JOB_ID_HTTP_HEADER + "' is expected");
JobContext context = jobManager.getJobContext(jobId);
String jobToken = request.getHeader(JobManager.JOB_TOKEN_HTTP_HEADER);
if (jobToken == null)
throw new OneException("Http header '" + JobManager.JOB_TOKEN_HTTP_HEADER + "' is expected");
JobContext context = jobManager.getJobContext(jobToken);
if (context == null)
throw new OneException("No job context found for specified job id");
throw new OneException("No job context found for specified job token");
return context;
}

1
server-plugin/server-plugin-maven/src/main/java/io/onedev/server/plugin/maven/DefaultMavenCISpecProvider.java
View File

@ -83,6 +83,7 @@ public class DefaultMavenCISpecProvider implements DefaultCISpecProvider {
* OneDev using extracted version for current build
*/
job.setCommands(""
+ "echo \"Detecting project version (may require some time while downloading maven dependencies)...\"\n"
+ "buildVersion=$(mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)\n"
+ "echo \"##onedev[SetBuildVersion '$buildVersion']\"\n"
+ "echo\n"

28
server-plugin/server-plugin-maven/src/main/java/io/onedev/server/plugin/maven/MavenLogNormalizer.java
View File

@ -1,28 +0,0 @@
package io.onedev.server.plugin.maven;
import javax.inject.Singleton;
import io.onedev.server.ci.job.log.LogLevel;
import io.onedev.server.ci.job.log.LogNormalizer;
@Singleton
public class MavenLogNormalizer implements LogNormalizer {
@Override
public Normalized normalize(String message) {
if (message.startsWith("[INFO] ")) {
return new Normalized(LogLevel.INFO, message.substring("[INFO] ".length()));
} else if (message.startsWith("[ERROR] ")) {
return new Normalized(LogLevel.ERROR, message.substring("[ERROR] ".length()));
} else if (message.startsWith("[WARNING] ")) {
return new Normalized(LogLevel.WARN, message.substring("[WARNING] ".length()));
} else if (message.startsWith("[DEBUG] ")) {
return new Normalized(LogLevel.DEBUG, message.substring("[DEBUG] ".length()));
} else if (message.startsWith("[TRACE] ")) {
return new Normalized(LogLevel.TRACE, message.substring("[TRACE] ".length()));
} else {
return null;
}
}
}

2
server-plugin/server-plugin-maven/src/main/java/io/onedev/server/plugin/maven/MavenModule.java
View File

@ -2,7 +2,6 @@ package io.onedev.server.plugin.maven;
import io.onedev.commons.launcher.loader.AbstractPluginModule;
import io.onedev.server.ci.DefaultCISpecProvider;
import io.onedev.server.ci.job.log.LogNormalizer;
/**
* NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@ -16,7 +15,6 @@ public class MavenModule extends AbstractPluginModule {
// put your guice bindings here
contribute(DefaultCISpecProvider.class, DefaultMavenCISpecProvider.class);
contribute(LogNormalizer.class, MavenLogNormalizer.class);
}
}

124
server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java
View File

@ -37,6 +37,7 @@ import io.onedev.server.ci.job.cache.CacheAllocation;
import io.onedev.server.ci.job.cache.CacheCallable;
import io.onedev.server.ci.job.cache.CacheRunner;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.ci.job.log.JobLogger;
import io.onedev.server.model.support.JobContext;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.plugin.serverdocker.ServerDockerExecutor.TestData;
@ -122,8 +123,8 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
@SuppressWarnings("unchecked")
private String getImageOS(Logger logger, String image) {
logger.info("Checking image OS...");
private String getImageOS(JobLogger logger, String image) {
logger.log("Checking image OS...");
Commandline docker = getDocker();
docker.addArgs("inspect", image);
@ -132,11 +133,11 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
@Override
public void consume(String line) {
logger.debug(line);
logger.log(line);
output.append(line).append("\n");
}
}, newErrorLogger(logger), logger).checkReturnCode();
}, newCommandLogger(logger)).checkReturnCode();
Map<String, Object> map;
try {
@ -159,36 +160,36 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
@Override
public void execute(String jobId, JobContext context) {
Logger logger = context.getLogger();
public void execute(String jobToken, JobContext jobContext) {
JobLogger logger = jobContext.getLogger();
getCapacityRunner().call(new Callable<Void>() {
@Override
public Void call() {
return new CacheRunner(getCacheHome(), context.getCaches()).call(new CacheCallable<Void>() {
return new CacheRunner(getCacheHome(), jobContext.getCaches()).call(new CacheCallable<Void>() {
@Override
public Void call(Collection<CacheAllocation> allocations) {
context.notifyJobRunning();
jobContext.notifyJobRunning();
login(logger);
logger.info("Pulling image...") ;
logger.log("Pulling image...") ;
Commandline docker = getDocker();
docker.addArgs("pull", context.getEnvironment());
docker.execute(newInfoLogger(logger), newErrorLogger(logger), logger).checkReturnCode();
docker.addArgs("pull", jobContext.getEnvironment());
docker.execute(newCommandLogger(logger), newCommandLogger(logger)).checkReturnCode();
docker.clearArgs();
String jobInstance = UUID.randomUUID().toString();
docker.addArgs("run", "--rm", "--name", jobInstance);
for (Map.Entry<String, String> entry: context.getEnvVars().entrySet())
for (Map.Entry<String, String> entry: jobContext.getEnvVars().entrySet())
docker.addArgs("--env", entry.getKey() + "=" + entry.getValue());
if (getRunOptions() != null)
docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
String imageOS = getImageOS(logger, context.getEnvironment());
logger.info("Detected image OS: " + imageOS);
String imageOS = getImageOS(logger, jobContext.getEnvironment());
logger.log("Detected image OS: " + imageOS);
boolean windows = imageOS.equals("windows");
@ -206,16 +207,16 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
}
File effectiveWorkspace = workspaceCache != null? workspaceCache: context.getServerWorkspace();
File effectiveWorkspace = workspaceCache != null? workspaceCache: jobContext.getServerWorkspace();
if (context.isCloneSource()) {
logger.info("Cloning source code...");
context.checkoutSource(effectiveWorkspace);
if (jobContext.isCloneSource()) {
logger.log("Cloning source code...");
jobContext.checkoutSource(effectiveWorkspace);
}
if (workspaceCache != null) {
try {
FileUtils.copyDirectory(context.getServerWorkspace(), workspaceCache);
FileUtils.copyDirectory(jobContext.getServerWorkspace(), workspaceCache);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -231,45 +232,45 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
if (windows) {
File scriptFile = new File(effectiveWorkspace, "onedev-job-commands.bat");
try {
FileUtils.writeLines(scriptFile, context.getCommands(), "\r\n");
FileUtils.writeLines(scriptFile, jobContext.getCommands(), "\r\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
docker.addArgs(context.getEnvironment());
docker.addArgs(jobContext.getEnvironment());
docker.addArgs("cmd", "/c", dockerWorkspacePath + "\\onedev-job-commands.bat");
} else {
File scriptFile = new File(effectiveWorkspace, "onedev-job-commands.sh");
try {
FileUtils.writeLines(scriptFile, context.getCommands(), "\n");
FileUtils.writeLines(scriptFile, jobContext.getCommands(), "\n");
} catch (IOException e) {
throw new RuntimeException(e);
}
docker.addArgs(context.getEnvironment());
docker.addArgs(jobContext.getEnvironment());
docker.addArgs("sh", dockerWorkspacePath + "/onedev-job-commands.sh");
}
logger.info("Running container to execute job...");
logger.log("Running container to execute job...");
try {
docker.execute(newInfoLogger(logger), newErrorLogger(logger), null, new ProcessKiller() {
docker.execute(newCommandLogger(logger), newCommandLogger(logger), null, new ProcessKiller() {
@Override
public void kill(Process process) {
logger.info("Stopping container...");
logger.log("Stopping container...");
Commandline cmd = getDocker();
cmd.addArgs("stop", jobInstance);
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), logger);
cmd.execute(newCommandLogger(logger), newCommandLogger(logger)).checkReturnCode();
}
}, logger).checkReturnCode();
}).checkReturnCode();
return null;
} finally {
if (workspaceCache != null) {
int baseLen = workspaceCache.getAbsolutePath().length()+1;
for (File file: context.getCollectFiles().listFiles(workspaceCache)) {
for (File file: jobContext.getCollectFiles().listFiles(workspaceCache)) {
try {
FileUtils.copyFile(file, new File(context.getServerWorkspace(), file.getAbsolutePath().substring(baseLen)));
FileUtils.copyFile(file, new File(jobContext.getServerWorkspace(), file.getAbsolutePath().substring(baseLen)));
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -278,40 +279,29 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
}
}, logger);
});
}
});
}
private LineConsumer newInfoLogger(Logger logger) {
private LineConsumer newCommandLogger(JobLogger logger) {
return new LineConsumer(Charsets.UTF_8.name()) {
@Override
public void consume(String line) {
logger.info(line);
logger.log(line);
}
};
}
private LineConsumer newErrorLogger(Logger logger) {
return new LineConsumer(Charsets.UTF_8.name()) {
@Override
public void consume(String line) {
logger.error(line);
}
};
}
private void login(Logger logger) {
private void login(JobLogger logger) {
for (RegistryLogin login: getRegistryLogins()) {
if (login.getRegistryUrl() != null)
logger.info("Login to docker registry '{}'...", login.getRegistryUrl());
logger.log(String.format("Login to docker registry '%s'...", login.getRegistryUrl()));
else
logger.info("Login to official docker registry...");
logger.log("Login to official docker registry...");
Commandline cmd = getDocker();
cmd.addArgs("login", "-u", login.getUserName(), "--password-stdin");
if (login.getRegistryUrl() != null)
@ -322,7 +312,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), input, logger).checkReturnCode();
cmd.execute(newCommandLogger(logger), newCommandLogger(logger), input).checkReturnCode();
}
}
@ -409,19 +399,27 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
@Override
public void test(TestData testData) {
logger.info("Testing local docker executor...");
JobLogger logger = new JobLogger() {
@Override
public void log(String message, Throwable t) {
ServerDockerExecutor.logger.info(message, t);
}
};
logger.log("Testing local docker executor...");
login(logger);
logger.info("Pulling image...");
logger.log("Pulling image...");
Commandline cmd = getDocker();
cmd.addArgs("pull", testData.getDockerImage());
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), logger).checkReturnCode();
cmd.execute(newCommandLogger(logger), newCommandLogger(logger)).checkReturnCode();
boolean windows = getImageOS(logger, testData.getDockerImage()).equals("windows");
logger.info("Running container...");
logger.log("Running container...");
File cacheHome = getCacheHome();
boolean cacheHomeExists = cacheHome.exists();
File workspaceDir = null;
@ -455,7 +453,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
else
cmd.addArgs("sh", "-c", "echo hello from container");
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), logger).checkReturnCode();
cmd.execute(newCommandLogger(logger), newCommandLogger(logger)).checkReturnCode();
} finally {
if (workspaceDir != null)
FileUtils.deleteDir(workspaceDir);
@ -466,10 +464,10 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
if (!SystemUtils.IS_OS_WINDOWS) {
logger.info("Checking busybox...");
logger.log("Checking busybox...");
cmd = getDocker();
cmd.addArgs("run", "--rm", "busybox", "sh", "-c", "echo hello from busybox");
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), logger).checkReturnCode();
cmd.execute(newCommandLogger(logger), newCommandLogger(logger)).checkReturnCode();
}
}
@ -482,7 +480,21 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
String containerPath = "/onedev_dir_to_clean";
cmd.addArgs("run", "-v", dir.getAbsolutePath() + ":" + containerPath, "--rm",
"busybox", "sh", "-c", "rm -rf " + containerPath + "/*");
cmd.execute(newInfoLogger(logger), newErrorLogger(logger), logger).checkReturnCode();
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.info(line);
}
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.error(line);
}
}).checkReturnCode();
}
}