archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

Use $home/onedev-cache to store job caches

Browse Source
This commit is contained in:
Robin Shen 2019-07-09 22:09:31 +08:00
parent 99248180a4
commit 9652fd1c83
6 changed files with 17 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server-core/src/main/java/io/onedev/server/ci/job/DefaultJobManager.java
View File

@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
import io.onedev.commons.launcher.loader.Listen;
@ -354,9 +355,10 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
});
} catch (Exception e) {
if (ExceptionUtils.find(e, InterruptedException.class) == null) {
DefaultJobManager.logger.debug("Error running build", e);
if (e.getMessage() != null)
logger.log(e.getMessage());
else
logger.log(Throwables.getStackTraceAsString(e));
}
String errorMessage = e.getMessage();
if (errorMessage != null) {

5
server-core/src/main/java/io/onedev/server/model/support/JobExecutor.java
View File

@ -1,5 +1,6 @@
package io.onedev.server.model.support;
import java.io.File;
import java.io.Serializable;
import org.eclipse.jgit.lib.ObjectId;
@ -106,6 +107,10 @@ public abstract class JobExecutor implements Serializable {
public void setCacheTTL(int cacheTTL) {
this.cacheTTL = cacheTTL;
}
protected File getCacheHome() {
return new File(System.getProperty("user.home"), "onedev-cache");
}
public abstract void execute(String jobToken, JobContext context);

7
server-core/src/main/java/io/onedev/server/storage/DefaultStorageManager.java
View File

@ -167,11 +167,4 @@ public class DefaultStorageManager implements StorageManager {
return buildDir;
}
@Override
public File getJobCacheDir() {
File projectsDir = new File(getStorageDir(), "jobcache");
FileUtils.createDir(projectsDir);
return projectsDir;
}
}

2
server-core/src/main/java/io/onedev/server/storage/StorageManager.java
View File

@ -49,6 +49,4 @@ public interface StorageManager {
*/
File getBuildDir(Long projectId, Long buildNumber);
File getJobCacheDir();
}

59
server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
View File

@ -14,10 +14,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import org.apache.commons.codec.Charsets;
import org.apache.commons.io.FilenameUtils;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -48,16 +46,13 @@ import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.plugin.kubernetes.KubernetesExecutor.TestData;
import io.onedev.server.util.JobLogger;
import io.onedev.server.util.inputspec.SecretInput;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.NameOfEmptyValue;
import io.onedev.server.web.editable.annotation.OmitName;
import io.onedev.server.web.util.Testable;
@Editable(order=300)
@ClassValidating
public class KubernetesExecutor extends JobExecutor implements Testable<TestData>, Validatable {
public class KubernetesExecutor extends JobExecutor implements Testable<TestData> {
private static final long serialVersionUID = 1L;
@ -85,8 +80,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
private String memoryRequest = "128m";
private String cacheHome;
@Editable(name="Kubectl Config File", order=100, description=
"Specify absolute path to the config file used by kubectl to access the "
+ "cluster. Leave empty to have kubectl determining cluster access "
@ -180,27 +173,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
this.memoryRequest = memoryRequest;
}
@Editable(order=40000, group="More Settings", description="Optionally specify an absolute directory on Kubernetes "
+ "working nodes to store job caches of this executor. If leave empty, OneDev will use directory "
+ "<tt>/onedev-cache</tt> on Linux, and <tt>C:\\onedev-cache</tt> on Windows")
@NameOfEmptyValue("Use default")
public String getCacheHome() {
return cacheHome;
}
public void setCacheHome(String cacheHome) {
this.cacheHome = cacheHome;
}
private String getEffectiveCacheHome(String osName) {
if (getCacheHome() != null)
return cacheHome;
else if (osName.equalsIgnoreCase("linux"))
return "/onedev-cache";
else
return "C:\\onedev-cache";
}
@Override
public void execute(String jobToken, JobContext jobContext) {
execute(jobContext.getEnvironment(), jobToken, jobContext.getLogger(), jobContext);
@ -359,7 +331,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
"weight", i,
"preference", Maps.newLinkedHashMap("matchExpressions",
Lists.<Object>newArrayList(Maps.newLinkedHashMap(
"key", "onedev-cache-" + cacheSpec.getKey(),
"key", CACHE_LABEL_PREFIX + cacheSpec.getKey(),
"operator", "In",
"values", Lists.newArrayList(String.valueOf(i)))))));
}
@ -410,7 +382,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
logger.log(String.format("OS of working node is '%s'", osName));
return osName;
} else {
throw new OneException("No applicable working nodes found for executor '" + getEffectiveCacheHome(osName) + "'");
throw new OneException("No applicable working nodes found");
}
}
@ -447,20 +419,17 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
String k8sHelperClassPath;
String containerCIHome;
String containerWorkspace;
String containerCacheHome;
if (osName.equalsIgnoreCase("linux")) {
containerCIHome = "/onedev-ci";
containerWorkspace = containerCIHome + "/workspace";
containerCacheHome = containerCIHome + "/cache";
k8sHelperClassPath = "/k8s-helper/*";
containerCacheHome = "/onedev-cache";
mainContainerSpec.put("command", Lists.newArrayList("sh"));
mainContainerSpec.put("args", Lists.newArrayList(containerCIHome + "/commands.sh"));
} else {
containerCIHome = "C:\\onedev-ci";
containerWorkspace = containerCIHome + "\\workspace";
containerCacheHome = containerCIHome + "\\cache";
k8sHelperClassPath = "C:\\k8s-helper\\*";
containerCacheHome = "C:\\onedev-cache";
mainContainerSpec.put("command", Lists.newArrayList("cmd"));
mainContainerSpec.put("args", Lists.newArrayList("/c", containerCIHome + "\\commands.bat"));
}
@ -474,7 +443,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
List<Object> volumeMounts = Lists.<Object>newArrayList(ciPathMount, cacheHomeMount);
mainContainerSpec.put("workingDir", containerWorkspace);
mainContainerSpec.put("volumeMounts", volumeMounts);
mainContainerSpec.put("resources", Maps.newLinkedHashMap("requests", Maps.newLinkedHashMap(
@ -530,7 +498,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
Map<Object, Object> cacheHomeVolume = Maps.newLinkedHashMap(
"name", "cache-home",
"hostPath", Maps.newLinkedHashMap(
"path", JsonEscape.escapeJson(getEffectiveCacheHome(osName)),
"path", JsonEscape.escapeJson(getCacheHome().getAbsolutePath()),
"type", "DirectoryOrCreate"));
podSpec.put("volumes", Lists.<Object>newArrayList(ciHomeVolume, cacheHomeVolume));
@ -1014,21 +982,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
}).checkReturnCode();
}
@Override
public boolean isValid(ConstraintValidatorContext context) {
boolean isValid = true;
if (getCacheHome() != null && FilenameUtils.getPrefixLength(getCacheHome()) == 0) {
isValid = false;
context.buildConstraintViolationWithTemplate("An absolute path is expected")
.addPropertyNode("cacheHome").addConstraintViolation();
}
if (!isValid)
context.disableDefaultConstraintViolation();
return isValid;
}
@Editable
public static class NodeSelectorEntry implements Serializable {

10
server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java
View File

@ -41,7 +41,6 @@ import io.onedev.server.ci.job.JobContext;
import io.onedev.server.ci.job.JobManager;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.plugin.serverdocker.ServerDockerExecutor.TestData;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.JobLogger;
import io.onedev.server.util.OneContext;
import io.onedev.server.util.validation.Validatable;
@ -172,6 +171,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
JobManager jobManager = OneDev.getInstance(JobManager.class);
File hostCacheHome = getCacheHome();
FileUtils.createDir(hostCacheHome);
logger.log("Allocating job caches...") ;
Map<CacheInstance, Date> cacheInstances = KubernetesHelper.getCacheInstances(hostCacheHome);
@ -406,10 +406,6 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
return isValid;
}
private File getCacheHome() {
return OneDev.getInstance(StorageManager.class).getJobCacheDir();
}
@Override
public void test(TestData testData, JobLogger logger) {
login(logger);
@ -438,10 +434,10 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
String containerCachePath;
if (windows) {
containerWorkspacePath = "C:\\onedev-ci\\workspace";
containerCachePath = "C:\\onedev-cache";
containerCachePath = "C:\\onedev-ci\\cache";
} else {
containerWorkspacePath = "/onedev-ci/workspace";
containerCachePath = "/onedev-cache";
containerCachePath = "/onedev-ci/cache";
}
cmd.addArgs("-v", workspaceDir.getAbsolutePath() + ":" + containerWorkspacePath);
cmd.addArgs("-v", cacheDir.getAbsolutePath() + ":" + containerCachePath);