feat: Rework job cache to store on server instead of local disk (#1732)

pom.xml

@@ -9,7 +9,7 @@
 		<version>1.2.2</version>
 	</parent>
 	<artifactId>server</artifactId>
-	<version>10.0.0</version>
+	<version>10.1.0</version>
 	<packaging>pom</packaging>
 	<build>
 		<finalName>${project.groupId}.${project.artifactId}-${project.version}</finalName>
@@ -630,10 +630,10 @@
 		</repository>
 	</repositories>
 	<properties>
-		<commons.version>2.8.1</commons.version>
-		<agent.version>1.10.2</agent.version>
+		<commons.version>2.8.3</commons.version>
+		<agent.version>1.10.4</agent.version>
 		<slf4j.version>2.0.9</slf4j.version>
-		<logback.version>1.3.12</logback.version>
+		<logback.version>1.4.14</logback.version>
 		<antlr.version>4.7.2</antlr.version>
 		<jetty.version>9.4.51.v20230217</jetty.version>
 		<wicket.version>7.18.0</wicket.version>

server-core/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<build>
 		<plugins>

server-core/src/main/java/io/onedev/server/CoreModule.java

@@ -52,6 +52,8 @@ import io.onedev.server.job.DefaultJobManager;
 import io.onedev.server.job.DefaultResourceAllocator;
 import io.onedev.server.job.JobManager;
 import io.onedev.server.job.ResourceAllocator;
+import io.onedev.server.entitymanager.JobCacheManager;
+import io.onedev.server.entitymanager.impl.DefaultJobCacheManager;
 import io.onedev.server.job.log.DefaultLogManager;
 import io.onedev.server.job.log.LogManager;
 import io.onedev.server.mail.DefaultMailManager;
@@ -216,6 +218,7 @@ public class CoreModule extends AbstractPluginModule {
 		bind(BuildManager.class).to(DefaultBuildManager.class);
 		bind(BuildDependenceManager.class).to(DefaultBuildDependenceManager.class);
 		bind(JobManager.class).to(DefaultJobManager.class);
+		bind(JobCacheManager.class).to(DefaultJobCacheManager.class);
 		bind(LogManager.class).to(DefaultLogManager.class);
 		bind(MailManager.class).to(DefaultMailManager.class);
 		bind(IssueManager.class).to(DefaultIssueManager.class);

server-core/src/main/java/io/onedev/server/OneDev.java

@@ -7,6 +7,7 @@ import io.onedev.commons.loader.AbstractPlugin;
 import io.onedev.commons.loader.AppLoader;
 import io.onedev.commons.loader.ManagedSerializedForm;
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.TarUtils;
 import io.onedev.k8shelper.KubernetesHelper;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.data.DataManager;
@@ -216,7 +217,7 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {
 				try (Response response = builder.get()) {
 					KubernetesHelper.checkStatus(response);
 					try (InputStream is = response.readEntity(InputStream.class)) {							
-						FileUtils.untar(is, getAssetsDir(), false);
+						TarUtils.untar(is, getAssetsDir(), false);
 					} catch (IOException e) {
 						throw new RuntimeException(e);
 					}

server-core/src/main/java/io/onedev/server/attachment/DefaultAttachmentManager.java

@@ -4,6 +4,7 @@ import io.onedev.commons.loader.ManagedSerializedForm;
 import io.onedev.commons.utils.ExceptionUtils;
 import io.onedev.commons.utils.FileUtils;
 import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TarUtils;
 import io.onedev.k8shelper.KubernetesHelper;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.cluster.ClusterRunnable;
@@ -285,7 +286,7 @@ public class DefaultAttachmentManager implements AttachmentManager, SchedulableT
 			try (Response response = builder.get()) {
 				KubernetesHelper.checkStatus(response);
 				try (InputStream is = response.readEntity(InputStream.class)) {
-					FileUtils.untar(is, targetDir, false);
+					TarUtils.untar(is, targetDir, false);
 				} catch (IOException e) {
 					throw new RuntimeException(e);
 				}

server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java

@@ -1798,5 +1798,22 @@ public class BuildSpec implements Serializable, Validatable {
 			}
 		}
 	}
+
+	private void migrate27(VersionedYamlDoc doc, Stack<Integer> versions) {
+		for (NodeTuple specTuple: doc.getValue()) {
+			String specObjectKey = ((ScalarNode) specTuple.getKeyNode()).getValue();
+			if (specObjectKey.equals("jobs")) {
+				SequenceNode jobsNode = (SequenceNode) specTuple.getValueNode();
+				for (Node jobsNodeItem : jobsNode.getValue()) {
+					MappingNode jobNode = (MappingNode) jobsNodeItem;
+					for (var itJobTuple = jobNode.getValue().iterator(); itJobTuple.hasNext();) {
+						String jobTupleKey = ((ScalarNode) itJobTuple.next().getKeyNode()).getValue();
+						if (jobTupleKey.equals("caches")) 
+							itJobTuple.remove();
+					}
+				}
+			}
+		}
+	}
 	
 }

server-core/src/main/java/io/onedev/server/buildspec/Service.java

@@ -101,7 +101,7 @@ public class Service implements NamedElement, Serializable {
 		this.readinessCheckCommand = readinessCheckCommand;
 	}
 
-	@Editable(order=500, name="Built-in Registry Access Token", description = "Specify access token for built-in docker registry if necessary")
+	@Editable(order=500, name="Built-in Registry Access Token", group="More Settings", description = "Specify access token for built-in docker registry if necessary")
 	@ChoiceProvider("getAccessTokenSecretChoices")
 	public String getBuiltInRegistryAccessTokenSecret() {
 		return builtInRegistryAccessTokenSecret;

server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java

@@ -1,74 +0,0 @@
-package io.onedev.server.buildspec.job;
-
-import java.io.Serializable;
-import java.util.List;
-
-import javax.validation.ConstraintValidatorContext;
-
-import javax.validation.constraints.NotEmpty;
-
-import io.onedev.commons.codeassist.InputSuggestion;
-import io.onedev.commons.utils.PathUtils;
-import io.onedev.server.buildspec.BuildSpec;
-import io.onedev.server.validation.Validatable;
-import io.onedev.server.annotation.ClassValidating;
-import io.onedev.server.annotation.RegEx;
-import io.onedev.server.annotation.Editable;
-import io.onedev.server.annotation.Interpolative;
-
-@Editable
-@ClassValidating
-public class CacheSpec implements Serializable, Validatable {
-
-	private static final long serialVersionUID = 1L;
-	
-	private String key;
-	
-	private String path;
-
-	@Editable(order=100, description="Specify key of the cache. Caches with same key can be reused by different projects/jobs. "
-			+ "Embed project/job variable to prevent cross project/job reuse")
-	@Interpolative(variableSuggester="suggestVariables")
-	@NotEmpty
-	@RegEx(pattern ="[a-zA-Z0-9\\-_\\.]+", message="Can only contain alphanumeric, dash, dot and underscore")
-	public String getKey() {
-		return key;
-	}
-
-	public void setKey(String key) {
-		this.key = key;
-	}
-	
-	public String getNormalizedKey() {
-		return getKey().replaceAll("[^a-zA-Z0-9\\-_\\.]", "-");
-	}
-	
-	@Editable(order=200, description="Specify path to cache. Non-absolute path is considered to be relative to job workspace. "
-			+ "Please note that shell executor only allows non-absolute path here")
-	@Interpolative(variableSuggester="suggestVariables")
-	@NotEmpty
-	public String getPath() {
-		return path;
-	}
-
-	public void setPath(String path) {
-		this.path = path;
-	}
-	
-	@SuppressWarnings("unused")
-	private static List<InputSuggestion> suggestVariables(String matchWith) {
-		return BuildSpec.suggestVariables(matchWith, false, false, false);
-	}
-
-	@Override
-	public boolean isValid(ConstraintValidatorContext context) {
-		if (PathUtils.isCurrent(path)) {
-			context.disableDefaultConstraintViolation();
-			context.buildConstraintViolationWithTemplate("Invalid path").addPropertyNode("path").addConstraintViolation();
-			return false;
-		} else {
-			return true;
-		}
-	}
-
-}

server-core/src/main/java/io/onedev/server/buildspec/job/Job.java

@@ -83,8 +83,6 @@ public class Job implements NamedElement, Serializable, Validatable {
 	private List<String> requiredServices = new ArrayList<>();
 	
 	private List<JobTrigger> triggers = new ArrayList<>();
-	
-	private List<CacheSpec> caches = new ArrayList<>();
 
 	private long timeout = 3600;
 	
@@ -277,22 +275,6 @@ public class Job implements NamedElement, Serializable, Validatable {
 		this.retryDelay = retryDelay;
 	}
 	
-	@Editable(order=10100, group="More Settings", description="Cache specific paths to speed up job execution. "
-			+ "For instance for Java Maven projects executed by various docker executors, you may cache folder "
-			+ "<tt>/root/.m2/repository</tt> to avoid downloading dependencies for subsequent executions.<br>"
-			+ "<b class='text-danger'>WARNING</b>: When using cache, malicious jobs running with same job executor "
-			+ "can read or even pollute the cache intentionally using same cache key as yours. To avoid this "
-			+ "issue, make sure job executor executing your job can only be used by trusted jobs via job "
-			+ "authorization setting</b>")
-	@Valid
-	public List<CacheSpec> getCaches() {
-		return caches;
-	}
-
-	public void setCaches(List<CacheSpec> caches) {
-		this.caches = caches;
-	}
-
 	@Editable(order=10500, group="More Settings", description="Specify timeout in seconds")
 	public long getTimeout() {
 		return timeout;
@@ -328,18 +310,6 @@ public class Job implements NamedElement, Serializable, Validatable {
 		
 		Set<String> keys = new HashSet<>();
 		Set<String> paths = new HashSet<>();
-		for (CacheSpec cache: caches) {
-			if (!keys.add(cache.getKey())) {
-				isValid = false;
-				context.buildConstraintViolationWithTemplate("Duplicate key (" + cache.getKey() + ")")
-						.addPropertyNode("caches").addConstraintViolation();
-			}
-			if (!paths.add(cache.getPath())) {
-				isValid = false;
-				context.buildConstraintViolationWithTemplate("Duplicate path (" + cache.getPath() + ")")
-						.addPropertyNode("caches").addConstraintViolation();
-			} 
-		}
 
 		Set<String> dependencyJobNames = new HashSet<>();
 		for (JobDependency dependency: jobDependencies) {

server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java

@@ -16,9 +16,10 @@ import io.onedev.server.util.UrlUtils;
 import javax.validation.constraints.NotEmpty;
 import java.util.List;
 
+import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;
 import static java.util.stream.Collectors.toList;
 
-@Editable(order=160, name="Build Docker Image", description="Build and publish docker image with docker buildx. " +
+@Editable(order=160, name="Build Docker Image", group = DOCKER_IMAGE, description="Build and publish docker image with docker buildx. " +
 		"This step can only be executed by server docker executor or remote docker executor. To build image with " +
 		"Kubernetes executor, please use kaniko step instead")
 public class BuildImageStep extends Step {
@@ -85,7 +86,7 @@ public class BuildImageStep extends Step {
 		this.publish = publish;
 	}
 
-	@Editable(order=335, name="Built-in Registry Access Token Secret", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
+	@Editable(order=1000, name="Built-in Registry Access Token Secret", group = "More Settings", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
 	@ChoiceProvider("getAccessTokenSecretChoices")
 	public String getBuiltInRegistryAccessTokenSecret() {
 		return builtInRegistryAccessTokenSecret;
@@ -107,7 +108,7 @@ public class BuildImageStep extends Step {
 				"<code>" + server + "</code>";
 	}
 	
-	@Editable(order=340, name="Remove Dangling Images After Build")
+	@Editable(order=1100, name="Remove Dangling Images After Build", group = "More Settings")
 	public boolean isRemoveDanglingImages() {
 		return removeDanglingImages;
 	}
@@ -116,7 +117,7 @@ public class BuildImageStep extends Step {
 		this.removeDanglingImages = removeDanglingImages;
 	}
 
-	@Editable(order=350, description="Optionally specify additional options to build image, " +
+	@Editable(order=1200, group = "More Settings", description="Optionally specify additional options to build image, " +
 			"separated by spaces. For instance <code>--builder</code> and <code>--platform</code> can be " +
 			"used to build multi-arch images")
 	@Interpolative(variableSuggester="suggestVariables")

server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java

@@ -20,9 +20,10 @@ import java.util.ArrayList;
 import java.util.List;
 
 import static io.onedev.agent.DockerExecutorUtils.buildDockerConfig;
+import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;
 import static java.util.stream.Collectors.toList;
 
-@Editable(order=200, name="Build Docker Image (Kaniko)", description="Build and publish docker image with Kaniko. " +
+@Editable(order=200, name="Build Docker Image (Kaniko)", group = DOCKER_IMAGE, description="Build and publish docker image with Kaniko. " +
 		"This step can be executed by server docker executor, remote docker executor, or Kubernetes executor, " +
 		"without the need to mount docker sock")
 public class BuildImageWithKanikoStep extends CommandStep {
@@ -80,7 +81,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
 		this.destinations = destinations;
 	}
 	
-	@Editable(order=325, name="Certificates to Trust", placeholder = "Base64 encoded PEM format, starting with " +
+	@Editable(order=1000, name="Certificates to Trust", group = "More Settings", placeholder = "Base64 encoded PEM format, starting with " +
 			"-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",
 			description = "Specify certificates to trust if you are using self-signed certificates for your docker registries")
 	@Multiline(monospace = true)
@@ -93,7 +94,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
 		this.trustCertificates = trustCertificates;
 	}
 
-	@Editable(order=340, name="Built-in Registry Access Token Secret", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
+	@Editable(order=1100, name="Built-in Registry Access Token Secret", group="More Settings", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
 	@ChoiceProvider("getAccessTokenSecretChoices")
 	@Override
 	public String getBuiltInRegistryAccessTokenSecret() {
@@ -112,7 +113,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
 				"<code>" + server + "</code>";
 	}
 	
-	@Editable(order=350, description="Optionally specify additional options to build image, " +
+	@Editable(order=1200, group="More Settings", description="Optionally specify additional options to build image, " +
 			"separated by spaces")
 	@Interpolative(variableSuggester="suggestVariables")
 	@ReservedOptions({"(--context)=.*", "(--destination)=.*"})

server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java

@@ -1,25 +1,24 @@
 package io.onedev.server.buildspec.step;
 
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import javax.validation.constraints.NotEmpty;
-
 import io.onedev.commons.codeassist.InputSuggestion;
 import io.onedev.commons.utils.ExplicitException;
 import io.onedev.commons.utils.TaskLogger;
 import io.onedev.server.OneDev;
+import io.onedev.server.annotation.ChoiceProvider;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
 import io.onedev.server.buildspec.BuildSpec;
 import io.onedev.server.entitymanager.MilestoneManager;
 import io.onedev.server.model.Build;
 import io.onedev.server.model.Milestone;
 import io.onedev.server.model.Project;
 import io.onedev.server.persistence.TransactionManager;
-import io.onedev.server.annotation.ChoiceProvider;
-import io.onedev.server.annotation.Editable;
-import io.onedev.server.annotation.Interpolative;
+
+import javax.validation.constraints.NotEmpty;
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 @Editable(name="Close Milestone", order=400)
 public class CloseMilestoneStep extends ServerSideStep {
@@ -46,10 +45,9 @@ public class CloseMilestoneStep extends ServerSideStep {
 		return BuildSpec.suggestVariables(matchWith, true, true, false);
 	}
 
-	@Editable(order=1060, description="Specify a secret to be used as access token. This access token " +
-			"should have permission to manage issues in the project")
+	@Editable(order=1060, description="For build commit not reachable from default branch, " +
+			"an access token with manage issue permission is required")
 	@ChoiceProvider("getAccessTokenSecretChoices")
-	@NotEmpty
 	public String getAccessTokenSecret() {
 		return accessTokenSecret;
 	}
@@ -75,11 +73,7 @@ public class CloseMilestoneStep extends ServerSideStep {
 				MilestoneManager milestoneManager = OneDev.getInstance(MilestoneManager.class);
 				Milestone milestone = milestoneManager.findInHierarchy(project, milestoneName);
 				if (milestone != null) {
-					// Access token is left empty if we migrate from old version
-					if (getAccessTokenSecret() == null)
-						throw new ExplicitException("Access token secret not specified");
-					
-					if (build.canCloseMilestone(getAccessTokenSecret(), milestoneName)) {
+					if (build.canCloseMilestone(getAccessTokenSecret())) {
 						milestone.setClosed(true);
 						milestoneManager.update(milestone);
 					} else {

server-core/src/main/java/io/onedev/server/buildspec/step/CommandStep.java

@@ -68,30 +68,7 @@ public class CommandStep extends Step {
 	public void setImage(String image) {
 		this.image = image;
 	}
-
-	@Editable(order=105, name="Built-in Registry Access Token Secret", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
-	@ChoiceProvider("getAccessTokenSecretChoices")
-	@ShowCondition("isRunInContainerEnabled")
-	public String getBuiltInRegistryAccessTokenSecret() {
-		return builtInRegistryAccessTokenSecret;
-	}
-
-	public void setBuiltInRegistryAccessTokenSecret(String builtInRegistryAccessTokenSecret) {
-		this.builtInRegistryAccessTokenSecret = builtInRegistryAccessTokenSecret;
-	}
-
-	protected static List<String> getAccessTokenSecretChoices() {
-		return Project.get().getHierarchyJobSecrets()
-				.stream().map(it->it.getName()).collect(Collectors.toList());
-	}
 	
-	private static String getBuiltInRegistryAccessTokenSecretDescription() {
-		var serverUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
-		var server = UrlUtils.getServer(serverUrl);
-		return "Optionally specify a secret to be used as access token for built-in registry server " +
-				"<code>" + server + "</code>";
-	}
-
 	static List<InputSuggestion> suggestVariables(String matchWith) {
 		return BuildSpec.suggestVariables(matchWith, false, false, false);
 	}
@@ -106,7 +83,31 @@ public class CommandStep extends Step {
 		this.interpreter = interpreter;
 	}
 
-	@Editable(order=10000, name="Enable TTY Mode", description=USE_TTY_HELP)
+	@Editable(order=9000, name="Built-in Registry Access Token Secret", group = "More Settings",
+			descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
+	@ChoiceProvider("getAccessTokenSecretChoices")
+	@ShowCondition("isRunInContainerEnabled")
+	public String getBuiltInRegistryAccessTokenSecret() {
+		return builtInRegistryAccessTokenSecret;
+	}
+
+	public void setBuiltInRegistryAccessTokenSecret(String builtInRegistryAccessTokenSecret) {
+		this.builtInRegistryAccessTokenSecret = builtInRegistryAccessTokenSecret;
+	}
+
+	protected static List<String> getAccessTokenSecretChoices() {
+		return Project.get().getHierarchyJobSecrets()
+				.stream().map(it->it.getName()).collect(Collectors.toList());
+	}
+
+	private static String getBuiltInRegistryAccessTokenSecretDescription() {
+		var serverUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
+		var server = UrlUtils.getServer(serverUrl);
+		return "Optionally specify a secret to be used as access token for built-in registry server " +
+				"<code>" + server + "</code>";
+	}
+	
+	@Editable(order=10000, name="Enable TTY Mode", group = "More Settings", description=USE_TTY_HELP)
 	@ShowCondition("isRunInContainerEnabled")
 	public boolean isUseTTY() {
 		return useTTY;

server-core/src/main/java/io/onedev/server/buildspec/step/CreateBranchStep.java

@@ -4,7 +4,10 @@ import io.onedev.commons.codeassist.InputSuggestion;
 import io.onedev.commons.utils.ExplicitException;
 import io.onedev.commons.utils.TaskLogger;
 import io.onedev.server.OneDev;
-import io.onedev.server.annotation.*;
+import io.onedev.server.annotation.BranchName;
+import io.onedev.server.annotation.ChoiceProvider;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
 import io.onedev.server.buildspec.BuildSpec;
 import io.onedev.server.git.GitUtils;
 import io.onedev.server.git.service.GitService;
@@ -68,11 +71,10 @@ public class CreateBranchStep extends ServerSideStep {
 		else
 			return new ArrayList<>();
 	}
-	
-	@Editable(order=1060, description="Specify a secret to be used as access token. This access token " +
-			"should have permission to create above branch in the project")
+
+	@Editable(order=1060, description="For build commit not reachable from default branch, " +
+			"an access token with create branch permission is required")
 	@ChoiceProvider("getAccessTokenSecretChoices")
-	@NotEmpty
 	public String getAccessTokenSecret() {
 		return accessTokenSecret;
 	}

server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java

@@ -61,10 +61,9 @@ public class CreateTagStep extends ServerSideStep {
 		return BuildSpec.suggestVariables(matchWith, true, true, false);
 	}
 
-	@Editable(order=1060, description="Specify a secret to be used as access token. This access token " +
-			"should have permission to create above tag in the project")
+	@Editable(order=1060, description="For build commit not reachable from default branch, " +
+			"an access token with create tag permission is required")
 	@ChoiceProvider("getAccessTokenSecretChoices")
-	@NotEmpty
 	public String getAccessTokenSecret() {
 		return accessTokenSecret;
 	}
@@ -87,10 +86,6 @@ public class CreateTagStep extends ServerSideStep {
 		
 		if (!Repository.isValidRefName(GitUtils.tag2ref(tagName)))
 			throw new ExplicitException("Invalid tag name: " + tagName);
-
-		// Access token is left empty if we migrate from old version
-		if (getAccessTokenSecret() == null)
-			throw new ExplicitException("Access token secret not specified");
 		
 		if (build.canCreateTag(getAccessTokenSecret(), tagName)) {
 			RefFacade tagRef = project.getTagRef(tagName);

server-core/src/main/java/io/onedev/server/buildspec/step/GenerateChecksumStep.java

@@ -0,0 +1,88 @@
+package io.onedev.server.buildspec.step;
+
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.buildspec.step.commandinterpreter.Interpreter;
+import io.onedev.server.buildspec.step.commandinterpreter.ShellInterpreter;
+
+import javax.validation.constraints.NotEmpty;
+import java.util.ArrayList;
+import java.util.List;
+
+import static io.onedev.server.buildspec.step.StepGroup.UTILITIES;
+
+@Editable(order=1110, group = UTILITIES, name="Generate File Checksum", description = "" +
+		"This step can only be executed by a docker aware executor")
+public class GenerateChecksumStep extends CommandStep {
+
+	private static final long serialVersionUID = 1L;
+	
+	private String files;
+
+	private String targetFile;
+	
+	@Editable(order=100, description = "Specify files to create sha256 checksum from. Multiple files " +
+			"should be separated by space. <a href='https://www.linuxjournal.com/content/globstar-new-bash-globbing-option' target='_blank'>Globstar</a> patterns accepted. " +
+			"Non-absolute file is considered to be relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a>")
+	@Interpolative(variableSuggester="suggestVariables")
+	@NotEmpty
+	public String getFiles() {
+		return files;
+	}
+
+	public void setFiles(String files) {
+		this.files = files;
+	}
+
+	@Editable(order=200, description = "Specify a file relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a> to write checksum into")
+	@Interpolative(variableSuggester="suggestVariables")
+	@NotEmpty
+	public String getTargetFile() {
+		return targetFile;
+	}
+
+	public void setTargetFile(String targetFile) {
+		this.targetFile = targetFile;
+	}
+
+	@Override
+	public boolean isRunInContainer() {
+		return true;
+	}
+
+	@Override
+	public String getImage() {
+		return "ubuntu:20.04";
+	}
+
+	@Override
+	public boolean isUseTTY() {
+		return false;
+	}
+
+	@Override
+	public String getBuiltInRegistryAccessTokenSecret() {
+		return null;
+	}
+
+	@Override
+	public Interpreter getInterpreter() {
+		return new ShellInterpreter() {
+			
+			@Override
+			public String getShell() {
+				return "bash";
+			}
+
+			@Override
+			public List<String> getCommands() {
+				var commands = new ArrayList<String>();
+				commands.add("set -e");
+				commands.add("shopt -s globstar");
+				commands.add("cat `ls -1 " + files + " 2>/dev/null` | sha256sum | awk '{ print $1 }' > " + targetFile);
+				return commands;
+			}
+		};
+	}
+	
+}

server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java

@@ -20,7 +20,9 @@ import java.io.File;
 import java.util.List;
 import java.util.Map;
 
-@Editable(order=1050, name="Publish Artifacts")
+import static io.onedev.server.buildspec.step.StepGroup.PUBLISH;
+
+@Editable(order=1050, group= PUBLISH, name="Artifacts")
 public class PublishArtifactStep extends ServerSideStep {
 
 	private static final long serialVersionUID = 1L;

server-core/src/main/java/io/onedev/server/buildspec/step/PublishSiteStep.java

@@ -29,7 +29,9 @@ import io.onedev.server.annotation.Interpolative;
 import io.onedev.server.annotation.Patterns;
 import io.onedev.server.annotation.ProjectChoice;
 
-@Editable(order=1060, name="Publish Site", description="This step publishes specified files to be served as project web site. "
+import static io.onedev.server.buildspec.step.StepGroup.PUBLISH;
+
+@Editable(order=1060, name="Site", group = PUBLISH, description="This step publishes specified files to be served as project web site. "
 		+ "Project web site can be accessed publicly via <code>http://&lt;onedev base url&gt;/path/to/project/~site</code>")
 public class PublishSiteStep extends ServerSideStep {
 

server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java

@@ -51,7 +51,7 @@ import static io.onedev.server.git.GitUtils.getReachableCommits;
 import static java.util.stream.Collectors.toList;
 import static org.eclipse.jgit.lib.Constants.R_HEADS;
 
-@Editable(order=60, name="Pull from Remote", group=StepGroup.REPOSITORY_SYNC, description=""
+@Editable(order=1070, name="Pull from Remote", group=StepGroup.REPOSITORY_SYNC, description=""
 		+ "This step pulls specified refs from remote. For security reason, it is only allowed "
 		+ "to run from default branch")
 public class PullRepository extends SyncRepository {

server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java

@@ -20,7 +20,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
-@Editable(order=70, name="Push to Remote", group=StepGroup.REPOSITORY_SYNC, 
+@Editable(order=1080, name="Push to Remote", group=StepGroup.REPOSITORY_SYNC, 
 		description="This step pushes current commit to same ref on remote")
 public class PushRepository extends SyncRepository {
 

server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java

@@ -8,7 +8,6 @@ import io.onedev.server.SubscriptionManager;
 import io.onedev.server.annotation.ChoiceProvider;
 import io.onedev.server.annotation.Editable;
 import io.onedev.server.annotation.Interpolative;
-import io.onedev.server.annotation.ShowCondition;
 import io.onedev.server.buildspec.BuildSpec;
 import io.onedev.server.buildspec.job.EnvVar;
 import io.onedev.server.buildspec.param.ParamCombination;
@@ -24,8 +23,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-@Editable(order=150, name="Run Docker Container", description="Run specified docker container. To access files in "
-		+ "job workspace, either use environment variable <tt>ONEDEV_WORKSPACE</tt>, or specify volume mounts. " +
+@Editable(order=150, name="Run Docker Container", description="Run specified docker container. <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>Job workspace</a> "
+		+ "is mounted into the container and its path is placed in environment variable <code>ONEDEV_WORKSPACE</code>. " +
 		"<b class='text-warning'>Note: </b> this step can only be executed by server docker executor or remote " +
 		"docker executor")
 public class RunContainerStep extends Step {
@@ -91,7 +90,7 @@ public class RunContainerStep extends Step {
 		this.envVars = envVars;
 	}
 
-	@Editable(order=500, description="Optionally mount directories or files under job workspace into container")
+	@Editable(order=500, group = "More Settings", description="Optionally mount directories or files under job workspace into container")
 	public List<VolumeMount> getVolumeMounts() {
 		return volumeMounts;
 	}
@@ -100,12 +99,11 @@ public class RunContainerStep extends Step {
 		this.volumeMounts = volumeMounts;
 	}
 
-	@Editable(order=600, name="Built-in Registry Access Token Secret", description="Optionally specify a " +
+	@Editable(order=600, name="Built-in Registry Access Token Secret", group = "More Settings", description="Optionally specify a " +
 			"access token secret to access built-in container registry if necessary. If this step needs to " +
 			"access external container registry, login information should be configured in corresponding " +
 			"executor then")
 	@ChoiceProvider("getAccessTokenSecretChoices")
-	@ShowCondition("isSubscriptionActive")
 	public String getBuiltInRegistryAccessTokenSecret() {
 		return builtInRegistryAccessTokenSecret;
 	}
@@ -123,7 +121,7 @@ public class RunContainerStep extends Step {
 		return OneDev.getInstance(SubscriptionManager.class).isSubscriptionActive();
 	}
 	
-	@Editable(order=10000, name="Enable TTY Mode", description="Many commands print outputs with ANSI colors in "
+	@Editable(order=10000, name="Enable TTY Mode", group = "More Settings", description="Many commands print outputs with ANSI colors in "
 			+ "TTY mode to help identifying problems easily. However some commands running in this mode may "
 			+ "wait for user input to cause build hanging. This can normally be fixed by adding extra options "
 			+ "to the command")

server-core/src/main/java/io/onedev/server/buildspec/step/RunImagetoolsStep.java

@@ -18,9 +18,10 @@ import io.onedev.server.util.UrlUtils;
 import javax.validation.constraints.NotEmpty;
 import java.util.List;
 
+import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;
 import static java.util.stream.Collectors.toList;
 
-@Editable(order=230, name="Run Docker Buildx Image Tools", description="Run docker buildx imagetools " +
+@Editable(order=230, name="Run Docker Buildx Image Tools", group = DOCKER_IMAGE, description="Run docker buildx imagetools " +
 		"command with specified arguments. This step can only be executed by server docker executor " +
 		"or remote docker executor")
 public class RunImagetoolsStep extends Step {
@@ -43,7 +44,7 @@ public class RunImagetoolsStep extends Step {
 		this.arguments = arguments;
 	}
 	
-	@Editable(order=200, name="Built-in Registry Access Token Secret", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
+	@Editable(order=200, name="Built-in Registry Access Token Secret", group = "More Settings", descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
 	@ChoiceProvider("getAccessTokenSecretChoices")
 	public String getBuiltInRegistryAccessTokenSecret() {
 		return builtInRegistryAccessTokenSecret;

server-core/src/main/java/io/onedev/server/buildspec/step/SCPCommandStep.java

@@ -14,8 +14,9 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import static com.google.common.collect.Lists.newArrayList;
+import static io.onedev.server.buildspec.step.StepGroup.UTILITIES;
 
-@Editable(order=135, name="Copy Files with SCP", description = "" +
+@Editable(order=1100, group = UTILITIES, name="Copy Files with SCP", description = "" +
 		"This step can only be executed by a docker aware executor. It runs under <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a>")
 public class SCPCommandStep extends CommandStep {
 
@@ -95,6 +96,11 @@ public class SCPCommandStep extends CommandStep {
 		return false;
 	}
 
+	@Override
+	public String getBuiltInRegistryAccessTokenSecret() {
+		return null;
+	}
+	
 	@Override
 	public Interpreter getInterpreter() {
 		return new DefaultInterpreter() {

server-core/src/main/java/io/onedev/server/buildspec/step/SSHCommandStep.java

@@ -16,8 +16,9 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import static com.google.common.collect.Lists.newArrayList;
+import static io.onedev.server.buildspec.step.StepGroup.UTILITIES;
 
-@Editable(order=125, name="Execute Commands via SSH", description = "" +
+@Editable(order=1090, group = UTILITIES, name="Execute Commands via SSH", description = "" +
 		"This step can only be executed by a docker aware executor")
 public class SSHCommandStep extends CommandStep {
 
@@ -114,6 +115,11 @@ public class SSHCommandStep extends CommandStep {
 		return false;
 	}
 
+	@Override
+	public String getBuiltInRegistryAccessTokenSecret() {
+		return null;
+	}
+	
 	@Override
 	public Interpreter getInterpreter() {
 		return new DefaultInterpreter() {

server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java

@@ -0,0 +1,111 @@
+package io.onedev.server.buildspec.step;
+
+import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.k8shelper.SetupCacheFacade;
+import io.onedev.k8shelper.StepFacade;
+import io.onedev.server.annotation.ChoiceProvider;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.buildspec.BuildSpec;
+import io.onedev.server.buildspec.param.ParamCombination;
+import io.onedev.server.model.Build;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;
+
+import javax.validation.constraints.NotEmpty;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.stream.Collectors.toList;
+
+@Editable(order=55, name="Set Up Cache", description = "Set up job cache to speed up job execution. " +
+		"Check <a href='https://docs.onedev.io/tutorials/cicd/job-cache' target='_blank'>this tutorial</a> " +
+		"to get familiar with job cache")
+public class SetupCacheStep extends Step {
+
+	private static final long serialVersionUID = 1L;
+
+	private String key;
+	
+	private List<String> loadKeys = new ArrayList<>();
+	
+	private String path;
+	
+	private String uploadAccessTokenSecret;
+
+	@Editable(order=100, name="Cache Key", description = "This key is used to determine if there is a cache hit. " +
+			"A cache is considered hit if its key is exactly the same as the key defined here")
+	@Interpolative(variableSuggester="suggestVariables")
+	@NotEmpty
+	public String getKey() {
+		return key;
+	}
+
+	public void setKey(String key) {
+		this.key = key;
+	}
+
+	@Editable(order=200, name="Cache Load Keys", description = "In case cache is not hit via above key, OneDev will " +
+			"loop through load keys defined here in order until a matching cache is found. A cache is considered " +
+			"matching if its key is prefixed with the load key. If multiple caches matches, the most recent cache " +
+			"will be returned")
+	@Interpolative(variableSuggester="suggestVariables")
+	public List<String> getLoadKeys() {
+		return loadKeys;
+	}
+
+	public void setLoadKeys(List<String> loadKeys) {
+		this.loadKeys = loadKeys;
+	}
+	
+	private static List<InputSuggestion> suggestVariables(String matchWith) {
+		return BuildSpec.suggestVariables(matchWith, true, true, false);
+	}
+	
+	@Editable(order=300, name="Cache Path", description = "For docker aware executors, this path is inside container, " +
+			"and accepts both absolute path and relative path (relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a>). " +
+			"For shell related executors which runs on host machine directly, only relative path is accepted")
+	@Interpolative(variableSuggester="suggestStaticVariables")
+	@NotEmpty
+	public String getPath() {
+		return path;
+	}
+
+	public void setPath(String path) {
+		this.path = path;
+	}
+
+	private static List<InputSuggestion> suggestStaticVariables(String matchWith) {
+		return BuildSpec.suggestVariables(matchWith, true, false, false);
+	}
+	
+	@Editable(order=500, description = "When build is successful, OneDev tries to upload caches not " +
+			"hit previously (note matching caches via load keys also considered not hit). This upload " +
+			"is permitted only when the build commit is reachable from default branch, or an access " +
+			"token is provided with upload cache permission")
+	@ChoiceProvider("getAccessTokenSecretChoices")
+	public String getUploadAccessTokenSecret() {
+		return uploadAccessTokenSecret;
+	}
+
+	public void setUploadAccessTokenSecret(String uploadAccessTokenSecret) {
+		this.uploadAccessTokenSecret = uploadAccessTokenSecret;
+	}
+
+	private static List<String> getAccessTokenSecretChoices() {
+		return Project.get().getHierarchyJobSecrets()
+				.stream().map(it->it.getName()).distinct().collect(toList());
+	}
+	
+	@Override
+	public StepFacade getFacade(Build build, JobExecutor jobExecutor, String jobToken, ParamCombination paramCombination) {
+		String accessToken;
+		if (getUploadAccessTokenSecret() != null)
+			accessToken = build.getJobAuthorizationContext().getSecretValue(getUploadAccessTokenSecret());
+		else
+			accessToken = null;
+		return new SetupCacheFacade(key, loadKeys, path, accessToken);
+	}
+	
+}
+ 

server-core/src/main/java/io/onedev/server/buildspec/step/StepGroup.java

@@ -2,8 +2,12 @@ package io.onedev.server.buildspec.step;
 
 public class StepGroup {
 
-	public static final String PUBLISH_REPORTS = "Publish Reports";
+	public static final String PUBLISH = "Publish";
 	
 	public static final String REPOSITORY_SYNC = "Repository Sync";
 	
+	public static final String UTILITIES = "Utilities";
+	
+	public static final String DOCKER_IMAGE = "Docker Image";
+	
 }

server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java

@@ -6,6 +6,7 @@ import io.onedev.server.OneDev;
 import io.onedev.server.StorageManager;
 import io.onedev.server.attachment.AttachmentManager;
 import io.onedev.server.entitymanager.BuildManager;
+import io.onedev.server.entitymanager.JobCacheManager;
 import io.onedev.server.entitymanager.PackBlobManager;
 import io.onedev.server.entitymanager.ProjectManager;
 import io.onedev.server.git.CommandUtils;
@@ -39,12 +40,12 @@ import java.io.*;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
-import static io.onedev.server.util.IOUtils.BUFFER_SIZE;
-import static io.onedev.commons.utils.FileUtils.tar;
+import static io.onedev.commons.utils.TarUtils.tar;
 import static io.onedev.commons.utils.LockUtils.read;
 import static io.onedev.commons.utils.LockUtils.write;
 import static io.onedev.server.model.Build.getArtifactsLockName;
 import static io.onedev.server.model.Project.SHARE_TEST_DIR;
+import static io.onedev.server.util.IOUtils.BUFFER_SIZE;
 import static javax.ws.rs.core.Response.Status.NO_CONTENT;
 import static javax.ws.rs.core.Response.ok;
 import static javax.ws.rs.core.Response.status;
@@ -69,13 +70,16 @@ public class ClusterResource {
 	
 	private final BuildManager buildManager;
 	
+	private final JobCacheManager jobCacheManager;
+	
 	private final WorkExecutor workExecutor;
 	
 	@Inject
 	public ClusterResource(ProjectManager projectManager, CommitInfoManager commitInfoManager, 
 						   AttachmentManager attachmentManager, VisitInfoManager visitInfoManager, 
 						   WorkExecutor workExecutor, StorageManager storageManager, 
-						   PackBlobManager packBlobManager, BuildManager buildManager) {
+						   PackBlobManager packBlobManager, BuildManager buildManager, 
+						   JobCacheManager jobCacheManager) {
 		this.commitInfoManager = commitInfoManager;
 		this.projectManager = projectManager;
 		this.workExecutor = workExecutor;
@@ -84,6 +88,7 @@ public class ClusterResource {
 		this.storageManager = storageManager;
 		this.packBlobManager = packBlobManager;
 		this.buildManager = buildManager;
+		this.jobCacheManager = jobCacheManager;
 	}
 
 	@Path("/project-files")
@@ -214,6 +219,18 @@ public class ClusterResource {
 		return ok(out).build();
 	}
 	
+	@Path("/cache")
+	@Produces(MediaType.APPLICATION_OCTET_STREAM)
+	@GET
+	public Response downloadCache(
+			@QueryParam("projectId") Long projectId, @QueryParam("cacheId") Long cacheId, 
+			@QueryParam("cachePath") String cachePath) {
+		if (!SecurityUtils.isSystem())
+			throw new UnauthorizedException("This api can only be accessed via cluster credential");
+		StreamingOutput out = os -> jobCacheManager.downloadCacheLocal(projectId, cacheId, cachePath, os, null);
+		return ok(out).build();
+	}
+
 	@Path("/site")
 	@Produces(MediaType.APPLICATION_OCTET_STREAM)
 	@GET
@@ -426,5 +443,19 @@ public class ClusterResource {
 			throw new RuntimeException(e);
 		}
 	}
+
+	@Path("/cache")
+	@Consumes(MediaType.APPLICATION_OCTET_STREAM)
+	@POST
+	public Response uploadCache(
+			@QueryParam("projectId") Long projectId,
+			@QueryParam("cacheId") Long cacheId,
+			@QueryParam("cachePath") String cachePath,
+			InputStream cacheStream) {
+		if (!SecurityUtils.isSystem())
+			throw new UnauthorizedException("This api can only be accessed via cluster credential");
+		jobCacheManager.uploadCacheLocal(projectId, cacheId, cachePath, cacheStream);
+		return ok().build();
+	}
 	
 }

server-core/src/main/java/io/onedev/server/commandhandler/BackupDatabase.java

@@ -4,6 +4,7 @@ import io.onedev.commons.bootstrap.Bootstrap;
 import io.onedev.commons.utils.ExceptionUtils;
 import io.onedev.commons.utils.ExplicitException;
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.ZipUtils;
 import io.onedev.server.persistence.HibernateConfig;
 import io.onedev.server.data.DataManager;
 import io.onedev.server.persistence.SessionFactoryManager;
@@ -78,7 +79,7 @@ public class BackupDatabase extends CommandHandler {
 				File tempDir = FileUtils.createTempDir("backup");
 				try {
 					dataManager.exportData(tempDir);
-					FileUtils.zip(tempDir, backupFile, null);
+					ZipUtils.zip(tempDir, backupFile, null);
 				} catch (Exception e) {
 					throw ExceptionUtils.unchecked(e);
 				} finally {

server-core/src/main/java/io/onedev/server/commandhandler/RestoreDatabase.java

@@ -4,6 +4,7 @@ import io.onedev.commons.bootstrap.Bootstrap;
 import io.onedev.commons.bootstrap.Command;
 import io.onedev.commons.utils.ExplicitException;
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.ZipUtils;
 import io.onedev.server.data.DataManager;
 import io.onedev.server.persistence.HibernateConfig;
 import io.onedev.server.persistence.SessionFactoryManager;
@@ -69,7 +70,7 @@ public class RestoreDatabase extends CommandHandler {
 				if (backupFile.isFile()) {
 					File dataDir = FileUtils.createTempDir("restore");
 					try {
-						FileUtils.unzip(backupFile, dataDir);
+						ZipUtils.unzip(backupFile, dataDir);
 						doRestore(dataDir);
 					} finally {
 						FileUtils.deleteDir(dataDir);

server-core/src/main/java/io/onedev/server/data/DefaultDataManager.java

@@ -5,10 +5,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import io.onedev.commons.bootstrap.Bootstrap;
 import io.onedev.commons.loader.ManagedSerializedForm;
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.FileUtils;
-import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.*;
 import io.onedev.server.OneDev;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.cluster.ClusterRunnable;
@@ -886,7 +883,7 @@ public class DefaultDataManager implements DataManager, Serializable {
 							exportData(tempDir);
 							File backupFile = new File(backupDir, 
 									DateTimeFormat.forPattern(Upgrade.BACKUP_DATETIME_FORMAT).print(new DateTime()) + ".zip");
-							FileUtils.zip(tempDir, backupFile, null);
+							ZipUtils.zip(tempDir, backupFile, null);
 						} catch (Exception e) {
 							notifyBackupError(e);
 							throw ExceptionUtils.unchecked(e);

server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java

@@ -6162,7 +6162,17 @@ public class DataMigrator {
 				dom.writeToFile(file, false);
 			}
 		}
-		
+	}
+
+	private void migrate154(File dataDir, Stack<Integer> versions) {
+		for (File file : dataDir.listFiles()) {
+			if (file.getName().startsWith("Roles.xml")) {
+				VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
+				for (Element element : dom.getRootElement().elements()) 
+					element.addElement("uploadCache").setText("false");
+				dom.writeToFile(file, false);
+			}
+		}
 	}
 	
 }

server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java

@@ -0,0 +1,40 @@
+package io.onedev.server.entitymanager;
+
+import io.onedev.server.model.JobCache;
+import io.onedev.server.model.Project;
+import io.onedev.server.persistence.dao.EntityManager;
+
+import javax.annotation.Nullable;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+public interface JobCacheManager extends EntityManager<JobCache> {
+	
+	void downloadCache(Long projectId, String cacheKey, String cachePath, OutputStream cacheStream);
+
+	boolean downloadCacheLocal(Long projectId, String cacheKey, String cachePath, 
+							Consumer<InputStream> cacheStreamHandler);
+	
+	void downloadCache(Long projectId, List<String> cacheLoadKey, String cachePath, OutputStream cacheStream);
+
+	boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKey, String cachePath, 
+							Consumer<InputStream> cacheStreamHandler);
+	
+	void downloadCacheLocal(Long projectId, Long cacheId, String cachePath, 
+							OutputStream cacheStream, @Nullable AtomicBoolean cacheHit);
+	
+	void uploadCache(Long projectId, String cacheKey, String cachePath, InputStream cacheStream);
+
+	void uploadCacheLocal(Long projectId, Long cacheId, String cachePath, InputStream cacheStream);
+
+	void uploadCacheLocal(Long projectId, String cacheKey, String cachePath, 
+						  Consumer<OutputStream> cacheStreamHandler);
+	
+	@Nullable
+	Long getCacheSize(Long projectId, Long cacheId);
+	
+}

server-core/src/main/java/io/onedev/server/entitymanager/ProjectManager.java

@@ -163,6 +163,8 @@ public interface ProjectManager extends EntityManager<Project> {
 	 * 			directory to store lucene index. The directory will be exist after calling this method
 	 */
 	File getIndexDir(Long projectId);
+	
+	File getCacheDir(Long projectId);
 
 	/**
 	 * Get directory to store static content of specified project

server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java

@@ -0,0 +1,444 @@
+package io.onedev.server.entitymanager.impl;
+
+import io.onedev.commons.loader.ManagedSerializedForm;
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.commons.utils.FileUtils;
+import io.onedev.k8shelper.KubernetesHelper;
+import io.onedev.server.cluster.ClusterManager;
+import io.onedev.server.entitymanager.JobCacheManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.event.Listen;
+import io.onedev.server.event.system.SystemStarted;
+import io.onedev.server.event.system.SystemStopping;
+import io.onedev.server.model.JobCache;
+import io.onedev.server.model.Project;
+import io.onedev.server.persistence.SessionManager;
+import io.onedev.server.persistence.TransactionManager;
+import io.onedev.server.persistence.annotation.Sessional;
+import io.onedev.server.persistence.annotation.Transactional;
+import io.onedev.server.persistence.dao.BaseEntityManager;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.taskschedule.SchedulableTask;
+import io.onedev.server.taskschedule.TaskScheduler;
+import io.onedev.server.util.IOUtils;
+import org.glassfish.jersey.client.ClientProperties;
+import org.hibernate.criterion.Restrictions;
+import org.hibernate.exception.ConstraintViolationException;
+import org.joda.time.DateTime;
+import org.quartz.CronScheduleBuilder;
+import org.quartz.ScheduleBuilder;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.*;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import static io.onedev.commons.utils.LockUtils.read;
+import static io.onedev.commons.utils.LockUtils.write;
+import static io.onedev.server.model.JobCache.*;
+import static io.onedev.server.util.IOUtils.BUFFER_SIZE;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Comparator.comparing;
+import static org.apache.commons.io.IOUtils.copy;
+
+@Singleton
+public class DefaultJobCacheManager extends BaseEntityManager<JobCache> 
+		implements JobCacheManager, Serializable, SchedulableTask {
+	
+	private static final int CACHE_VERSION = 1;
+	
+	private final ProjectManager projectManager;
+	
+	private final SessionManager sessionManager;
+	
+	private final TransactionManager transactionManager;
+	
+	private final ClusterManager clusterManager;
+	
+	private final TaskScheduler taskScheduler;
+	
+	private volatile String taskId;
+	
+	@Inject
+	public DefaultJobCacheManager(Dao dao, ProjectManager projectManager, SessionManager sessionManager, 
+								  TransactionManager transactionManager, ClusterManager clusterManager, 
+								  TaskScheduler taskScheduler) {
+		super(dao);
+		this.projectManager = projectManager;
+		this.sessionManager = sessionManager;
+		this.transactionManager = transactionManager;
+		this.clusterManager = clusterManager;
+		this.taskScheduler = taskScheduler;
+	}
+	
+	@Sessional
+	@Nullable
+	protected JobCache find(Project project, String cacheKey) {
+		var criteria = newCriteria();
+		criteria.add(Restrictions.eq(PROP_PROJECT, project));
+		criteria.add(Restrictions.eq(PROP_KEY, cacheKey));
+		return find(criteria);
+	}
+	
+	private String readString(File file) {
+		try {
+			return FileUtils.readFileToString(file, UTF_8);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+	
+	private void writeString(File file, String content) {
+		try {
+			FileUtils.writeStringToFile(file, content, UTF_8);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private void writeStream(OutputStream os, int value) {
+		try {
+			os.write(value);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+	
+	private Long getDownloadCacheId(Long projectId, String cacheKey) {
+		return sessionManager.call(() -> {
+			var project = projectManager.load(projectId);
+			var cache = find(project, cacheKey);
+			if (cache != null)
+				return cache.getId();
+			else
+				return null;
+		});
+	}
+	
+	@Override
+	public void downloadCache(Long projectId, String cacheKey, String cachePath, OutputStream cacheStream) {
+		var cacheId = getDownloadCacheId(projectId, cacheKey);
+		if (cacheId != null) 
+			downloadCache(projectId, cacheId, cachePath, cacheStream);
+		else 
+			writeStream(cacheStream, 0);
+	}
+	
+	private boolean downloadCacheLocal(Long projectId, Long cacheId, String cachePath, 
+									   Consumer<InputStream> cacheStreamHandler) {
+		return read(JobCache.getLockName(projectId, cacheId), () -> {
+			var is = openCacheInputStream(projectId, cacheId, cachePath);
+			if (is != null) try (is) {
+				cacheStreamHandler.accept(is);
+				return true;
+			} else {
+				return false;
+			}
+		});
+	}
+
+	@Override
+	public boolean downloadCacheLocal(Long projectId, String cacheKey, String cachePath, 
+								   Consumer<InputStream> cacheStreamHandler) {
+		var cacheId = getDownloadCacheId(projectId, cacheKey);
+		if (cacheId != null) 
+			return downloadCacheLocal(projectId, cacheId, cachePath, cacheStreamHandler);
+		else 
+			return false;
+	}
+	
+	private Long getDownloadCacheId(Long projectId, List<String> cacheLoadKeys) {
+		return sessionManager.call(() -> {
+			var project = projectManager.load(projectId);
+			for (var cacheLoadKey: cacheLoadKeys) {
+				var cache = project.getJobCaches().stream()
+						.filter(it->it.getKey().startsWith(cacheLoadKey))
+						.max(comparing(JobCache::getAccessDate));
+				if (cache.isPresent())
+					return cache.get().getId();
+			}
+			return null;
+		});			
+	}
+	
+	@Override
+	public void downloadCache(Long projectId, List<String> cacheLoadKeys, String cachePath, 
+							  OutputStream cacheStream) {
+		var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);
+		if (cacheId != null)
+			downloadCache(projectId, cacheId, cachePath, cacheStream);
+		else
+			writeStream(cacheStream, 0);
+	}
+
+	@Override
+	public boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKeys, String cachePath, 
+								   Consumer<InputStream> cacheStreamHandler) {
+		var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);
+		if (cacheId != null)
+			return downloadCacheLocal(projectId, cacheId, cachePath, cacheStreamHandler);
+		else
+			return false;
+	}
+	
+	@Nullable
+	private InputStream openCacheInputStream(Long projectId, Long cacheId, String cachePath) {
+		var cacheHome = projectManager.getCacheDir(projectId);
+		var cacheDir = new File(cacheHome, String.valueOf(cacheId));
+		if (cacheDir.exists()) {
+			var stamp = readString(new File(cacheDir, "stamp"));
+			if (stamp.equals(CACHE_VERSION + ":" + cachePath)) {
+				try {
+					return new FileInputStream(new File(cacheDir, "data"));
+				} catch (FileNotFoundException e) {
+					throw new RuntimeException(e);
+				}
+			}
+		}
+		return null;
+	}
+
+	@Override
+	public void downloadCacheLocal(Long projectId, Long cacheId, String cachePath, 
+								   OutputStream cacheStream, @Nullable AtomicBoolean cacheHit) {
+		read(JobCache.getLockName(projectId, cacheId), () -> {
+			var is = openCacheInputStream(projectId, cacheId, cachePath);
+			if (is != null) {
+				try (is) {
+					writeStream(cacheStream, 1);
+					if (cacheHit != null)
+						cacheHit.set(true);
+					else
+						writeStream(cacheStream, 1);
+					IOUtils.copy(is, cacheStream, BUFFER_SIZE);
+				} catch (IOException e) {
+					throw new RuntimeException(e);
+				}
+			} else {
+				writeStream(cacheStream, 0);
+				if (cacheHit != null)
+					cacheHit.set(false);
+				else
+					writeStream(cacheStream, 0);
+			}
+		});
+	}
+	
+	private void downloadCache(Long projectId, Long cacheId, String cachePath, OutputStream cacheStream) {
+		var localServer = clusterManager.getLocalServerAddress();
+		var activeServer = projectManager.getActiveServer(projectId, true);
+		boolean found;
+		if (localServer.equals(activeServer)) {
+			var cacheHit = new AtomicBoolean();
+			downloadCacheLocal(projectId, cacheId, cachePath, cacheStream, cacheHit);
+			found = cacheHit.get();
+		} else {
+			Client client = ClientBuilder.newClient();
+			client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
+			try {
+				String serverUrl = clusterManager.getServerUrl(activeServer);
+				var target = client.target(serverUrl)
+						.path("~api/cluster/cache")
+						.queryParam("projectId", projectId)
+						.queryParam("cacheId", cacheId)
+						.queryParam("cachePath", cachePath);
+				Invocation.Builder builder = target.request();
+				builder.header(HttpHeaders.AUTHORIZATION,
+						KubernetesHelper.BEARER + " " + clusterManager.getCredential());
+				try (Response response = builder.get()) {
+					KubernetesHelper.checkStatus(response);
+					try (var is = response.readEntity(InputStream.class)) {
+						found = is.read() == 1;
+						IOUtils.copy(is, cacheStream, BUFFER_SIZE);
+					} catch (IOException e) {
+						throw new RuntimeException(e);
+					}
+				}
+			} finally {
+				client.close();
+			}
+		}
+
+		if (found) {
+			transactionManager.run(() -> {
+				var cache = load(cacheId);
+				cache.setAccessDate(new Date());
+			});
+		};
+	}
+	
+	private Long getUploadCacheId(Long projectId, String cacheKey) {
+		Long cacheId;
+		while (true) {
+			try {
+				cacheId = transactionManager.call(() -> {
+					var project = projectManager.load(projectId);
+					var cache = find(project, cacheKey);
+					if (cache == null) {
+						cache = new JobCache();
+						cache.setProject(project);
+						cache.setKey(cacheKey);
+					}
+					cache.setAccessDate(new Date());
+					dao.persist(cache);
+					return cache.getId();
+				});
+				break;
+			} catch (Exception e) {
+				if (ExceptionUtils.find(e, ConstraintViolationException.class) != null) {
+					try {
+						Thread.sleep(5000);
+					} catch (InterruptedException ex) {
+						throw new RuntimeException(ex);
+					}
+				} else {
+					throw e;
+				}
+			}
+		}
+		return cacheId;
+	}
+	
+	@Override
+	public void uploadCache(Long projectId, String cacheKey, String cachePath, InputStream cacheStream) {
+		Long cacheId = getUploadCacheId(projectId, cacheKey);
+		
+		var localServer = clusterManager.getLocalServerAddress();
+		var activeServer = projectManager.getActiveServer(projectId, true);
+		if (localServer.equals(activeServer)) {
+			uploadCacheLocal(projectId, cacheId, cachePath, cacheStream);
+		} else {
+			Client client = ClientBuilder.newClient();
+			client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
+			try {
+				String serverUrl = clusterManager.getServerUrl(activeServer);
+				var target = client.target(serverUrl)
+						.path("~api/cluster/cache")
+						.queryParam("projectId", projectId)
+						.queryParam("cacheId", cacheId)
+						.queryParam("cachePath", cachePath);
+				Invocation.Builder builder = target.request();
+				builder.header(HttpHeaders.AUTHORIZATION,
+						KubernetesHelper.BEARER + " " + clusterManager.getCredential());
+				StreamingOutput output = os -> copy(cacheStream, os, BUFFER_SIZE);
+				try (Response response = builder.post(Entity.entity(output, MediaType.APPLICATION_OCTET_STREAM))) {
+					KubernetesHelper.checkStatus(response);
+				}
+			} finally {
+				client.close();
+			}
+		}
+	}
+	
+	private OutputStream openCacheOutputStream(Long projectId, Long cacheId, String cachePath) {
+		var cacheHome = projectManager.getCacheDir(projectId);
+		var cacheDir = new File(cacheHome, String.valueOf(cacheId));
+		FileUtils.createDir(cacheDir);
+		writeString(new File(cacheDir, "stamp"), CACHE_VERSION + ":" + cachePath);
+		try {
+			return new FileOutputStream(new File(cacheDir, "data"));
+		} catch (FileNotFoundException e) {
+			throw new RuntimeException(e);
+		}
+	} 
+
+	@Override
+	public void uploadCacheLocal(Long projectId, String cacheKey, String cachePath, 
+								 Consumer<OutputStream> cacheStreamHandler) {
+		Long cacheId = getUploadCacheId(projectId, cacheKey);
+		write(JobCache.getLockName(projectId, cacheId), () -> {
+			try (var os = openCacheOutputStream(projectId, cacheId, cachePath)) {
+				cacheStreamHandler.accept(os);
+			} catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+		});
+	}
+	
+	@Override
+	public void uploadCacheLocal(Long projectId, Long cacheId, String cachePath, InputStream cacheStream) {
+		write(JobCache.getLockName(projectId, cacheId), () -> {
+			try (var os = openCacheOutputStream(projectId, cacheId, cachePath)) {
+				IOUtils.copy(cacheStream, os, BUFFER_SIZE);	
+			} catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+		});
+	}
+
+	@Transactional
+	@Override
+	public void delete(JobCache cache) {
+		var projectId = cache.getProject().getId();
+		var cacheId = cache.getId();
+		dao.remove(cache);
+		projectManager.runOnActiveServer(projectId, () -> write(JobCache.getLockName(projectId, cacheId), () -> {
+			FileUtils.deleteDir(new File(projectManager.getCacheDir(projectId), String.valueOf(cacheId)));
+			return null;
+		}));
+	}
+
+	@Nullable
+	@Override
+	public Long getCacheSize(Long projectId, Long cacheId) {
+		return projectManager.runOnActiveServer(projectId, () -> read(JobCache.getLockName(projectId, cacheId), () -> {
+			var cacheDir = new File(projectManager.getCacheDir(projectId), String.valueOf(cacheId));
+			if (cacheDir.exists()) {
+				var stamp = readString(new File(cacheDir, "stamp"));
+				if (stamp.startsWith(CACHE_VERSION + ":")) {
+					var dataFile = new File(cacheDir, "data");
+					if (dataFile.exists())
+						return dataFile.length();
+				}
+			}
+			return null;
+		}));
+	}
+
+	public Object writeReplace() throws ObjectStreamException {
+		return new ManagedSerializedForm(JobCacheManager.class);
+	}
+
+	@Listen
+	public void on(SystemStarted event) {
+		taskId = taskScheduler.schedule(this);	
+	}
+	
+	@Listen
+	public void on(SystemStopping event) {
+		taskScheduler.unschedule(taskId);
+	}
+
+	@Transactional
+	@Override
+	public void execute() {
+		var now = new DateTime();
+		for (var project: projectManager.query()) {
+			var preserveDays = project.getHierarchyCachePreserveDays();
+			var threshold = now.minusDays(preserveDays);
+			var criteria = newCriteria();
+			criteria.add(Restrictions.eq(PROP_PROJECT, project));
+			criteria.add(Restrictions.lt(PROP_ACCESS_DATE, threshold.toDate()));
+			for (var cache: query(criteria)) 
+				delete(cache);
+		}
+	}
+
+	@Override
+	public ScheduleBuilder<?> getScheduleBuilder() {
+		return CronScheduleBuilder.dailyAtHourAndMinute(2, 30);
+	}
+	
+}

server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java

@@ -7,10 +7,7 @@ import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.map.IMap;
 import io.onedev.commons.bootstrap.Bootstrap;
 import io.onedev.commons.loader.ManagedSerializedForm;
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.FileUtils;
-import io.onedev.commons.utils.LockUtils;
+import io.onedev.commons.utils.*;
 import io.onedev.commons.utils.command.Commandline;
 import io.onedev.commons.utils.command.LineConsumer;
 import io.onedev.k8shelper.KubernetesHelper;
@@ -1654,7 +1651,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
 				try (Response response = builder.get()) {
 					KubernetesHelper.checkStatus(response);
 					try (InputStream is = response.readEntity(InputStream.class)) {
-						FileUtils.untar(is, directory, false);
+						TarUtils.untar(is, directory, false);
 					} catch (IOException e) {
 						throw new RuntimeException(e);
 					}
@@ -1863,7 +1860,12 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
 	public File getIndexDir(Long projectId) {
 		return getSubDir(projectId, "index");
 	}
-	
+
+	@Override
+	public File getCacheDir(Long projectId) {
+		return getSubDir(projectId, "cache");
+	}
+
 	@Override
 	public File getSiteDir(Long projectId) {
 		return getSubDir(projectId, SITE_DIR);

server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java

@@ -14,7 +14,10 @@ import io.onedev.server.annotation.Interpolative;
 import io.onedev.server.buildspec.BuildSpec;
 import io.onedev.server.buildspec.BuildSpecParseException;
 import io.onedev.server.buildspec.Service;
-import io.onedev.server.buildspec.job.*;
+import io.onedev.server.buildspec.job.Job;
+import io.onedev.server.buildspec.job.JobDependency;
+import io.onedev.server.buildspec.job.JobExecutorDiscoverer;
+import io.onedev.server.buildspec.job.TriggerMatch;
 import io.onedev.server.buildspec.job.action.PostBuildAction;
 import io.onedev.server.buildspec.job.action.condition.ActionCondition;
 import io.onedev.server.buildspec.job.projectdependency.ProjectDependency;
@@ -97,7 +100,6 @@ import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
-import java.util.function.Consumer;
 
 import static io.onedev.k8shelper.KubernetesHelper.BUILD_VERSION;
 import static io.onedev.k8shelper.KubernetesHelper.replacePlaceholders;
@@ -513,7 +515,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
 
 		AtomicInteger maxRetries = new AtomicInteger(0);
 		AtomicInteger retryDelay = new AtomicInteger(0);
-		List<CacheSpec> caches = new ArrayList<>();
 		List<ServiceFacade> services = new ArrayList<>();
 		List<Action> actions = new ArrayList<>();
 		long timeout;
@@ -529,9 +530,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
 				actions.add(step.getAction(build, jobExecutor, jobToken, build.getParamCombination()));
 			}
 
-			for (CacheSpec cache : job.getCaches())
-				caches.add(interpolator.interpolateProperties(cache));
-
 			for (String serviceName : job.getRequiredServices()) {
 				Service service = buildSpec.getServiceMap().get(serviceName);
 				services.add(interpolator.interpolateProperties(service).getFacade(build, jobToken));
@@ -551,8 +549,8 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
 			AtomicInteger retried = new AtomicInteger(0);
 			while (true) {
 				JobContext jobContext = new JobContext(jobToken, jobExecutor, projectId, projectPath,
-						projectGitDir, buildId, buildNumber, actions, refName, commitId, caches, 
-						services, timeout, retried.get());
+						projectGitDir, buildId, buildNumber, actions, refName, commitId, services, 
+						timeout, retried.get());
 				// Store original job actions as the copy in job context will be fetched from cluster and 
 				// some transient fields (such as step object in ServerSideFacade) will not be preserved 
 				jobActions.put(jobToken, actions);
@@ -1279,65 +1277,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
 		return false;
 	}
 
-	@Override
-	public Map<CacheInstance, String> allocateCaches(JobContext jobContext, CacheAllocationRequest request) {
-		String jobToken = jobContext.getJobToken();
-		return clusterManager.runOnServer(clusterManager.getLeaderServerAddress(), () -> {
-			synchronized (allocatedCaches) {
-				JobContext innerJobContext = getJobContext(jobToken, true);
-
-				List<CacheInstance> sortedInstances = new ArrayList<>(request.getInstances().keySet());
-				sortedInstances.sort((o1, o2) -> request.getInstances().get(o2).compareTo(request.getInstances().get(o1)));
-
-				Collection<String> allAllocated = new HashSet<>();
-				var activeJobTokens = getActiveJobTokens();
-				Collection<String> removeKeys = new HashSet<>();
-				for (var entry : allocatedCaches.entrySet()) {
-					if (activeJobTokens.contains(entry.getKey()))
-						allAllocated.addAll(entry.getValue());
-					else
-						removeKeys.add(entry.getKey());
-				}
-				for (var key : removeKeys)
-					allocatedCaches.remove(key);
-
-				Map<CacheInstance, String> allocations = new HashMap<>();
-
-				Collection<String> allocatedCachesOfJob = new ArrayList<>();
-				for (CacheSpec cacheSpec : innerJobContext.getCacheSpecs()) {
-					Optional<CacheInstance> result = sortedInstances
-							.stream()
-							.filter(it -> it.getCacheKey().equals(cacheSpec.getNormalizedKey()))
-							.filter(it -> !allAllocated.contains(it.getCacheUUID()))
-							.findFirst();
-					CacheInstance allocation;
-					allocation = result.orElseGet(() -> new CacheInstance(cacheSpec.getNormalizedKey(), UUID.randomUUID().toString()));
-					allocations.put(allocation, cacheSpec.getPath());
-					allocatedCachesOfJob.add(allocation.getCacheUUID());
-					allAllocated.add(allocation.getCacheUUID());
-				}
-
-				Consumer<CacheInstance> deletionMarker = instance -> {
-					long ellapsed = request.getCurrentTime().getTime() - request.getInstances().get(instance).getTime();
-					if (ellapsed > innerJobContext.getJobExecutor().getCacheTTL() * 24L * 3600L * 1000L) {
-						allocations.put(instance, null);
-						allocatedCachesOfJob.add(instance.getCacheUUID());
-						allAllocated.add(instance.getCacheUUID());
-					}
-				};
-
-				allocatedCaches.put(jobToken, allocatedCachesOfJob);
-
-				request.getInstances().keySet()
-						.stream()
-						.filter(it -> !allAllocated.contains(it.getCacheUUID()))
-						.forEach(deletionMarker);
-
-				return allocations;
-			}
-		});
-	}
-
 	@Override
 	public void runJob(String server, ClusterRunnable runnable) {
 		Future<?> future = null;
@@ -1466,7 +1405,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
 						try (Response response = builder.get()) {
 							KubernetesHelper.checkStatus(response);
 							try (InputStream is = response.readEntity(InputStream.class)) {
-								FileUtils.untar(is, targetDir, false);
+								TarUtils.untar(is, targetDir, false);
 							} catch (IOException e) {
 								throw new RuntimeException(e);
 							}

server-core/src/main/java/io/onedev/server/job/JobContext.java

@@ -3,13 +3,10 @@ package io.onedev.server.job;
 import io.onedev.k8shelper.Action;
 import io.onedev.k8shelper.LeafFacade;
 import io.onedev.k8shelper.ServiceFacade;
-import io.onedev.server.buildspec.Service;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;
 import org.eclipse.jgit.lib.ObjectId;
 
 import java.io.Serializable;
-import java.util.Collection;
 import java.util.List;
 
 public class JobContext implements Serializable {
@@ -36,18 +33,17 @@ public class JobContext implements Serializable {
 	
 	private final ObjectId commitId;
 	
-	private final Collection<CacheSpec> cacheSpecs; 
-	
 	private final List<ServiceFacade> services;
 	
 	private final long timeout;
 	
 	private final int retried;
 	
-	public JobContext(String jobToken, JobExecutor jobExecutor, Long projectId, String projectPath, 
-			String projectGitDir, Long buildId, Long buildNumber, List<Action> actions, 
-			String refName, ObjectId commitId, Collection<CacheSpec> caches, 
-			List<ServiceFacade> services, long timeout, int retried) {
+	public JobContext(String jobToken, JobExecutor jobExecutor, Long projectId, 
+					  String projectPath, String projectGitDir, Long buildId, 
+					  Long buildNumber, List<Action> actions, String refName, 
+					  ObjectId commitId, List<ServiceFacade> services, 
+					  long timeout, int retried) {
 		this.jobToken = jobToken;
 		this.jobExecutor = jobExecutor;
 		this.projectId = projectId;
@@ -58,7 +54,6 @@ public class JobContext implements Serializable {
 		this.actions = actions;
 		this.refName = refName;
 		this.commitId = commitId;
-		this.cacheSpecs = caches;
 		this.services = services;
 		this.timeout = timeout;
 		this.retried = retried;
@@ -87,11 +82,7 @@ public class JobContext implements Serializable {
 	public ObjectId getCommitId() {
 		return commitId;
 	}
-
-	public Collection<CacheSpec> getCacheSpecs() {
-		return cacheSpecs;
-	}
-
+	
 	public List<ServiceFacade> getServices() {
 		return services;
 	}

server-core/src/main/java/io/onedev/server/job/JobManager.java

@@ -1,8 +1,6 @@
 package io.onedev.server.job;
 
 import io.onedev.commons.utils.TaskLogger;
-import io.onedev.k8shelper.CacheAllocationRequest;
-import io.onedev.k8shelper.CacheInstance;
 import io.onedev.server.cluster.ClusterRunnable;
 import io.onedev.server.model.*;
 import io.onedev.server.terminal.Shell;
@@ -47,8 +45,6 @@ public interface JobManager {
 	
 	@Nullable
 	JobContext getJobContext(Long buildId);
-	
-	Map<CacheInstance, String> allocateCaches(JobContext jobContext, CacheAllocationRequest request);
 
 	void copyDependencies(JobContext jobContext, File targetDir);
 	

server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java

@@ -0,0 +1,114 @@
+package io.onedev.server.job;
+
+import io.onedev.commons.utils.TarUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.k8shelper.CacheHelper;
+import io.onedev.k8shelper.KubernetesHelper;
+import io.onedev.server.OneDev;
+import io.onedev.server.cluster.ClusterManager;
+import io.onedev.server.entitymanager.JobCacheManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.persistence.SessionManager;
+import io.onedev.server.security.SecurityUtils;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.List;
+
+public class ServerCacheHelper extends CacheHelper {
+	
+	private final JobContext jobContext;
+	
+	private final String localServer;
+	
+	public ServerCacheHelper(File buildHome, JobContext jobContext, TaskLogger logger) {
+		super(buildHome, logger);
+		this.jobContext = jobContext;
+		localServer = getClusterManager().getLocalServerAddress();
+	}
+
+	@Nullable
+	private String getRemoteCacheServerUrl() {
+		var activeServer = getProjectManager().getActiveServer(jobContext.getProjectId(), true);
+		if (!activeServer.equals(localServer)) 
+			return getClusterManager().getServerUrl(activeServer);
+		else 
+			return null;
+	}
+	
+	@Override
+	protected boolean downloadCache(String cacheKey, String cachePath, File cacheDir) {
+		var remoteCacheServerUrl = getRemoteCacheServerUrl();
+		if (remoteCacheServerUrl != null) {
+			return KubernetesHelper.downloadCache(remoteCacheServerUrl, jobContext.getJobToken(),
+					cacheKey, cachePath, cacheDir, null);
+		} else {
+			return getCacheManager().downloadCacheLocal(jobContext.getProjectId(),
+					cacheKey, cachePath, is -> TarUtils.untar(is, cacheDir, true));
+		}
+	}
+
+	@Override
+	protected boolean downloadCache(List<String> cacheLoadKeys, String cachePath, File cacheDir) {
+		var remoteCacheServerUrl = getRemoteCacheServerUrl();
+		if (remoteCacheServerUrl != null) {
+			return KubernetesHelper.downloadCache(remoteCacheServerUrl, jobContext.getJobToken(),
+					cacheLoadKeys, cachePath, cacheDir, null);
+		} else {
+			return getCacheManager().downloadCacheLocal(jobContext.getProjectId(), 
+					cacheLoadKeys, cachePath, is -> TarUtils.untar(is, cacheDir, true));
+		}
+	}
+
+	@Override
+	protected boolean uploadCache(String cacheKey, String cachePath,
+								  @Nullable String accessToken, File cacheDir) {
+		var remoteCacheServerUrl = getRemoteCacheServerUrl();
+		if (remoteCacheServerUrl != null) {
+			return KubernetesHelper.uploadCache(remoteCacheServerUrl, jobContext.getJobToken(),
+					cacheKey, cachePath, accessToken, cacheDir, null);
+		} else {
+			var authorized = getSessionManager().call(() -> {
+				var projectId = jobContext.getProjectId();
+				var project = getProjectManager().load(projectId);
+				if (project.isCommitOnBranch(jobContext.getCommitId(), project.getDefaultBranch())) {
+					return true;
+				} else if (accessToken != null) {
+					var user = getUserManager().findByAccessToken(accessToken);
+					return user != null && SecurityUtils.canUploadCache(user.asSubject(), project);
+				} else {
+					return false;
+				}
+			});
+			if (authorized) {
+				getCacheManager().uploadCacheLocal(jobContext.getProjectId(), cacheKey, 
+						cachePath, os -> TarUtils.tar(cacheDir, os, true));
+				return true;
+			} else {
+				return false;
+			}
+		}
+	}
+	
+	private SessionManager getSessionManager() {
+		return OneDev.getInstance(SessionManager.class);
+	}
+	
+	private ProjectManager getProjectManager() {
+		return OneDev.getInstance(ProjectManager.class);
+	}
+	
+	private UserManager getUserManager() {
+		return OneDev.getInstance(UserManager.class);
+	}
+	
+	private JobCacheManager getCacheManager() {
+		return OneDev.getInstance(JobCacheManager.class);
+	}
+	
+	private ClusterManager getClusterManager() {
+		return OneDev.getInstance(ClusterManager.class);
+	}
+	
+}

server-core/src/main/java/io/onedev/server/model/Build.java

@@ -911,10 +911,15 @@ public class Build extends ProjectBelonging
 	}
 
 	public boolean canCreateBranch(String accessTokenSecret, String branchName) {
-		return SecurityUtils.canCreateBranch(getUser(accessTokenSecret), getProject(), branchName);
+		var project = getProject();
+		return project.isCommitOnBranch(getCommitId(), project.getDefaultBranch())
+				|| accessTokenSecret != null && SecurityUtils.canCreateBranch(getUser(accessTokenSecret), project, branchName);
 	}
-	public boolean canCreateTag(String accessTokenSecret, String tagName) {
-		return SecurityUtils.canCreateTag(getUser(accessTokenSecret), getProject(), tagName);
+	
+	public boolean canCreateTag(@Nullable String accessTokenSecret, String tagName) {
+		var project = getProject();
+		return project.isCommitOnBranch(getCommitId(), project.getDefaultBranch())
+				|| accessTokenSecret != null && SecurityUtils.canCreateTag(getUser(accessTokenSecret), project, tagName);
 	}
 	
 	private User getUser(String accessTokenSecret) {
@@ -925,8 +930,10 @@ public class Build extends ProjectBelonging
 		return user;
 	}
 	
-	public boolean canCloseMilestone(String accessTokenSecret, String milestoneName) {
-		return SecurityUtils.canManageIssues(getUser(accessTokenSecret), getProject());
+	public boolean canCloseMilestone(@Nullable String accessTokenSecret) {
+		var project = getProject();
+		return project.isCommitOnBranch(getCommitId(), project.getDefaultBranch())
+				|| accessTokenSecret != null && SecurityUtils.canManageIssues(getUser(accessTokenSecret), project);
 	}
 	
 	public boolean isValid() {

server-core/src/main/java/io/onedev/server/model/JobCache.java

@@ -0,0 +1,65 @@
+package io.onedev.server.model;
+
+import javax.persistence.*;
+
+import java.util.Date;
+
+import static io.onedev.server.model.JobCache.PROP_ACCESS_DATE;
+import static io.onedev.server.model.JobCache.PROP_KEY;
+
+/**
+ * @author robin
+ *
+ */
+@Entity
+@Table(
+		indexes={@Index(columnList="o_project_id"), @Index(columnList=PROP_KEY), @Index(columnList = PROP_ACCESS_DATE)}, 
+		uniqueConstraints={@UniqueConstraint(columnNames={"o_project_id", PROP_KEY})})
+public class JobCache extends AbstractEntity {
+
+	private static final long serialVersionUID = 1L;
+	
+	public static final String PROP_PROJECT = "project";
+	
+	public static final String PROP_KEY = "key";
+	
+	public static final String PROP_ACCESS_DATE = "accessDate";
+	
+	@ManyToOne(fetch=FetchType.LAZY)
+	@JoinColumn(nullable=false)
+	private Project project;
+	
+	@Column(nullable=false)
+	private String key;
+	
+	private Date accessDate;
+
+	public Project getProject() {
+		return project;
+	}
+
+	public void setProject(Project project) {
+		this.project = project;
+	}
+
+	public String getKey() {
+		return key;
+	}
+
+	public void setKey(String key) {
+		this.key = key;
+	}
+
+	public Date getAccessDate() {
+		return accessDate;
+	}
+
+	public void setAccessDate(Date accessDate) {
+		this.accessDate = accessDate;
+	}
+
+	public static String getLockName(Long projectId, Long cacheId) {
+		return "job-cache:" + projectId + ":" + cacheId;
+	}
+	
+}

server-core/src/main/java/io/onedev/server/model/Project.java

@@ -215,6 +215,9 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
     @OneToMany(mappedBy="project")
     private Collection<Build> builds = new ArrayList<>();
 
+	@OneToMany(mappedBy="project", cascade=CascadeType.REMOVE)
+	private Collection<JobCache> jobCaches = new ArrayList<>();
+	
 	@OneToMany(mappedBy= "project")
 	private Collection<PackBlob> packBlobs = new ArrayList<>();
 	
@@ -1080,6 +1083,16 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
 		return null;
 	}
 	
+	public int getHierarchyCachePreserveDays() {
+		var cachePreserveDays = getBuildSetting().getCachePreserveDays();
+		if (cachePreserveDays != null)
+			return cachePreserveDays;
+		else if (getParent() != null)
+			return getParent().getHierarchyCachePreserveDays();
+		else 
+			return ProjectBuildSetting.DEFAULT_CACHE_PRESERVE_DAYS; 
+	}
+	
 	public ProjectPullRequestSetting getPullRequestSetting() {
 		return pullRequestSetting;
 	}
@@ -1178,6 +1191,14 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
 		this.builds = builds;
 	}
 
+	public Collection<JobCache> getJobCaches() {
+		return jobCaches;
+	}
+
+	public void setJobCaches(Collection<JobCache> jobCaches) {
+		this.jobCaches = jobCaches;
+	}
+
 	public Collection<PackBlob> getPackBlobs() {
 		return packBlobs;
 	}

server-core/src/main/java/io/onedev/server/model/Role.java

@@ -67,6 +67,8 @@ public class Role extends AbstractEntity implements BasePermission {
 	
 	private boolean manageBuilds;
 	
+	private boolean uploadCache;
+	
 	@Lob
 	@Column(length=65535, nullable=false)
 	private ArrayList<JobPrivilege> jobPrivileges = new ArrayList<>();
@@ -265,7 +267,18 @@ public class Role extends AbstractEntity implements BasePermission {
 	private static boolean isManageBuildsDisabled() {
 		return !(boolean)EditContext.get().getInputValue("manageBuilds");
 	}
-	
+
+	@Editable(order=675, description = "Enable to allow to upload build cache generated during CI/CD job. " +
+			"Uploaded cache can be used by subsequent builds of the project as long as cache key matches")
+	@ShowCondition("isManageBuildsDisabled")
+	public boolean isUploadCache() {
+		return uploadCache;
+	}
+
+	public void setUploadCache(boolean uploadCache) {
+		this.uploadCache = uploadCache;
+	}
+
 	@Editable(order=700)
 	@ShowCondition("isManageBuildsDisabled")
 	public List<JobPrivilege> getJobPrivileges() {
@@ -348,6 +361,8 @@ public class Role extends AbstractEntity implements BasePermission {
 			permissions.add(new EditIssueLink(linkAuthorization.getLink()));
 		if (manageBuilds)
 			permissions.add(new ManageBuilds());
+		if (uploadCache)
+			permissions.add(new UploadCache());
 		for (var jobPrivilege: jobPrivileges) {
 			permissions.add(new JobPermission(jobPrivilege.getJobNames(), new AccessBuild()));
 			if (jobPrivilege.isManageJob()) 

server-core/src/main/java/io/onedev/server/model/support/build/ProjectBuildSetting.java

@@ -20,6 +20,8 @@ import java.util.*;
 public class ProjectBuildSetting implements Serializable {
 
 	private static final long serialVersionUID = 1L;
+	
+	public static final int DEFAULT_CACHE_PRESERVE_DAYS = 7;
 
 	private List<String> listParams;
 	
@@ -33,6 +35,8 @@ public class ProjectBuildSetting implements Serializable {
 	
 	private List<DefaultFixedIssueFilter> defaultFixedIssueFilters = new ArrayList<>();
 	
+	private Integer cachePreserveDays;
+	
 	private transient GlobalBuildSetting globalSetting;
 	
 	public List<JobProperty> getJobProperties() {
@@ -67,6 +71,14 @@ public class ProjectBuildSetting implements Serializable {
 		this.defaultFixedIssueFilters = defaultFixedIssueFilters;
 	}
 
+	public Integer getCachePreserveDays() {
+		return cachePreserveDays;
+	}
+
+	public void setCachePreserveDays(Integer cachePreserveDays) {
+		this.cachePreserveDays = cachePreserveDays;
+	}
+
 	private GlobalBuildSetting getGlobalSetting() {
 		if (globalSetting == null)
 			globalSetting = OneDev.getInstance(SettingManager.class).getBuildSetting();

server-core/src/main/java/io/onedev/server/security/SecurityUtils.java

@@ -226,6 +226,14 @@ public class SecurityUtils extends org.apache.shiro.SecurityUtils {
 		return getSubject().isPermitted(new ProjectPermission(build.getProject(), 
 				new JobPermission(build.getJobName(), new ManageJob())));
 	}
+
+	public static boolean canUploadCache(Project project) {
+		return canUploadCache(getSubject(), project);
+	}
+
+	public static boolean canUploadCache(Subject subject, Project project) {
+		return subject.isPermitted(new ProjectPermission(project, new UploadCache()));
+	}
 	
 	public static boolean canAccessLog(Build build) {
 		return getSubject().isPermitted(new ProjectPermission(build.getProject(), 

server-core/src/main/java/io/onedev/server/security/permission/ManageBuilds.java

@@ -8,7 +8,8 @@ public class ManageBuilds implements BasePermission {
 
 	@Override
 	public boolean implies(Permission p) {
-		return p instanceof ManageBuilds || new JobPermission("*", new ManageJob()).implies(p);
+		return p instanceof ManageBuilds || new UploadCache().implies(p) 
+				|| new JobPermission("*", new ManageJob()).implies(p);
 	}
 
 	@Override

server-core/src/main/java/io/onedev/server/security/permission/UploadCache.java

@@ -0,0 +1,19 @@
+package io.onedev.server.security.permission;
+
+import io.onedev.server.util.facade.UserFacade;
+import org.apache.shiro.authz.Permission;
+import org.jetbrains.annotations.Nullable;
+
+public class UploadCache implements BasePermission {
+
+	@Override
+	public boolean implies(Permission p) {
+		return p instanceof UploadCache;
+	}
+
+	@Override
+	public boolean isApplicable(@Nullable UserFacade user) {
+		return user != null && !user.isEffectiveGuest();
+	}
+	
+}

server-core/src/main/java/io/onedev/server/web/asset/bootstrap/css/bootstrap-custom.css

@@ -337,7 +337,7 @@ kbd {
 }
 code {
     background-color: var(--light);
-    padding: 0.15rem 0.5rem;
+    padding: 0.15rem 0.25rem;
     font-weight: 400;
     border-radius: 0.42rem;
 }

server-core/src/main/java/io/onedev/server/web/component/polymorphiceditor/PolymorphicEditor.java

@@ -138,7 +138,7 @@ public class PolymorphicEditor extends ValueEditor<Serializable> {
 
 			@Override
 			protected void onUpdate(AjaxRequestTarget target) {
-				onTypeChanged(target);
+				onTypeChanging(target);
 				target.add(typeSelectorContainer.get("typeDescription"));
 				Component beanEditor = get("beanEditor");
 				target.add(beanEditor);
@@ -226,7 +226,7 @@ public class PolymorphicEditor extends ValueEditor<Serializable> {
 		return new HashSet<>();
 	}
 	
-	protected void onTypeChanged(AjaxRequestTarget target) {
+	protected void onTypeChanging(AjaxRequestTarget target) {
 	}
 	
 }

server-core/src/main/java/io/onedev/server/web/component/select2/Settings.java

@@ -12,17 +12,16 @@
  */
 package io.onedev.server.web.component.select2;
 
-import java.io.Serializable;
-import java.lang.reflect.Method;
-
-import org.json.JSONException;
-import org.json.JSONStringer;
-
+import io.onedev.server.annotation.OmitName;
 import io.onedev.server.util.ComponentContext;
 import io.onedev.server.web.component.select2.json.Json;
 import io.onedev.server.web.editable.EditableUtils;
 import io.onedev.server.web.editable.PropertyDescriptor;
-import io.onedev.server.annotation.OmitName;
+import org.json.JSONException;
+import org.json.JSONStringer;
+
+import java.io.Serializable;
+import java.lang.reflect.Method;
 
 /**
  * Select2 settings. Refer to the Select2 documentation for what these options
@@ -415,17 +414,16 @@ public final class Settings implements Serializable {
 		ComponentContext.push(new ComponentContext(select2));
 		try {
 			Method propertyGetter = propertyDescriptor.getPropertyGetter();
-			if (propertyDescriptor.isPropertyRequired()) {
+			String placeholder = EditableUtils.getPlaceholder(propertyGetter);
+			if (placeholder != null) {
+				setPlaceholder(placeholder);
+			} else if (propertyDescriptor.isPropertyRequired()) {
 				if (propertyDescriptor.getPropertyGetter().getAnnotation(OmitName.class) != null)
 					setPlaceholder("Choose " + propertyDescriptor.getDisplayName().toLowerCase() + "...");
 				else
 					setPlaceholder("Choose...");
 			} else if (propertyDescriptor.getPropertyGetter().getAnnotation(OmitName.class) != null) {
 				setPlaceholder(EditableUtils.getDisplayName(propertyDescriptor.getPropertyGetter()));
-			} else {
-				String placeholder = EditableUtils.getPlaceholder(propertyGetter);
-				if (placeholder != null)
-					setPlaceholder(placeholder);
 			}
 		} finally {
 			ComponentContext.pop();

server-core/src/main/java/io/onedev/server/web/component/typeselect/TypeSelectPanel.java

@@ -39,6 +39,7 @@ import io.onedev.server.web.asset.selectbytyping.SelectByTypingResourceReference
 import io.onedev.server.web.behavior.OnTypingDoneBehavior;
 import io.onedev.server.web.component.link.ViewStateAwareAjaxLink;
 import io.onedev.server.web.editable.EditableUtils;
+import org.unbescape.html.HtmlEscape;
 
 @SuppressWarnings("serial")
 public abstract class TypeSelectPanel<T extends Serializable> extends Panel {

server-core/src/main/java/io/onedev/server/web/editable/interpolativestringlist/InterpolativeStringListEditSupport.java

@@ -0,0 +1,67 @@
+package io.onedev.server.web.editable.interpolativestringlist;
+
+import io.onedev.commons.utils.ClassUtils;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.util.ReflectionUtils;
+import io.onedev.server.web.editable.*;
+import org.apache.wicket.Component;
+import org.apache.wicket.model.IModel;
+
+import java.io.Serializable;
+import java.lang.reflect.AnnotatedElement;
+import java.util.List;
+
+import static io.onedev.server.util.ReflectionUtils.getCollectionElementClass;
+
+@SuppressWarnings("serial")
+public class InterpolativeStringListEditSupport implements EditSupport {
+
+	@Override
+	public PropertyContext<?> getEditContext(PropertyDescriptor descriptor) {
+		if (List.class.isAssignableFrom(descriptor.getPropertyClass())) {
+			var propertyGetter = descriptor.getPropertyGetter();
+			Class<?> elementClass = getCollectionElementClass(propertyGetter.getGenericReturnType());
+			if (elementClass == String.class && propertyGetter.getAnnotation(Interpolative.class) != null) {
+				return new PropertyContext<List<String>>(descriptor) {
+
+					@Override
+					public PropertyViewer renderForView(String componentId, final IModel<List<String>> model) {
+						return new PropertyViewer(componentId, descriptor) {
+
+							@Override
+							protected Component newContent(String id, PropertyDescriptor propertyDescriptor) {
+								if (model.getObject() != null && !model.getObject().isEmpty()) {
+									return new InterpolativeStringListPropertyViewer(id, propertyDescriptor, model.getObject());
+								} else {
+									return new EmptyValueLabel(id) {
+
+										@Override
+										protected AnnotatedElement getElement() {
+											return propertyDescriptor.getPropertyGetter();
+										}
+										
+									};
+								}
+							}
+							
+						};
+					}
+
+					@Override
+					public PropertyEditor<List<String>> renderForEdit(String componentId, IModel<List<String>> model) {
+						return new InterpolativeStringListPropertyEditor(componentId, descriptor, model);
+					}
+					
+				};
+			}
+		}
+		return null;
+	}
+
+	@Override
+	public int getPriority() {
+		return DEFAULT_PRIORITY;
+	}
+	
+}

server-core/src/main/java/io/onedev/server/web/editable/interpolativestringlist/InterpolativeStringListPropertyEditor.html

@@ -0,0 +1,28 @@
+<wicket:panel>
+    <div class="string-list">
+        <table wicket:id="table">
+            <tbody>
+                <tr wicket:id="elements">
+					<td class="actions minimum">
+						<wicket:svg href="grip" class="icon drag-indicator"/>
+					</td>
+					<td>
+						<div class="clearable-wrapper">
+							<input wicket:id="elementEditor" type="text" class="form-control">
+						</div>
+						<div wicket:id="feedback"></div>
+					</td>
+					<td class="actions minimum">
+						<a wicket:id="deleteElement" class="btn btn-icon btn-light btn-hover-danger" title="Delete this"><wicket:svg href="trash" class="icon delete"/></a>
+					</td>
+                </tr>
+            </tbody>
+			<tfoot wicket:id="noRecords">
+				<tr>
+					<td colspan="3">Not defined</td>
+				</tr>
+			</tfoot>
+        </table>
+		<div class="foot"><a wicket:id="addElement" title="Add new" class="add-element btn btn-light btn-hover-primary btn-block"><wicket:svg href="plus" class="icon"></wicket:svg></a></div>
+    </div>
+</wicket:panel>

server-core/src/main/java/io/onedev/server/web/editable/interpolativestringlist/InterpolativeStringListPropertyEditor.java

@@ -0,0 +1,298 @@
+package io.onedev.server.web.editable.interpolativestringlist;
+
+import com.google.common.collect.Lists;
+import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.commons.utils.ClassUtils;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.util.Path;
+import io.onedev.server.util.PathNode;
+import io.onedev.server.util.ReflectionUtils;
+import io.onedev.server.web.behavior.InterpolativeAssistBehavior;
+import io.onedev.server.web.behavior.NoRecordsBehavior;
+import io.onedev.server.web.behavior.OnTypingDoneBehavior;
+import io.onedev.server.web.behavior.sortable.SortBehavior;
+import io.onedev.server.web.behavior.sortable.SortPosition;
+import io.onedev.server.web.editable.PropertyDescriptor;
+import io.onedev.server.web.editable.PropertyEditor;
+import io.onedev.server.web.editable.PropertyUpdating;
+import org.apache.wicket.Component;
+import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.ajax.markup.html.form.AjaxButton;
+import org.apache.wicket.behavior.AttributeAppender;
+import org.apache.wicket.event.IEvent;
+import org.apache.wicket.feedback.FencedFeedbackPanel;
+import org.apache.wicket.markup.ComponentTag;
+import org.apache.wicket.markup.html.WebMarkupContainer;
+import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.markup.html.form.TextField;
+import org.apache.wicket.markup.repeater.RepeatingView;
+import org.apache.wicket.model.IModel;
+import org.apache.wicket.model.Model;
+import org.apache.wicket.util.convert.ConversionException;
+
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+@SuppressWarnings("serial")
+public class InterpolativeStringListPropertyEditor extends PropertyEditor<List<String>> {
+	
+	private RepeatingView rows;
+
+	private WebMarkupContainer noRecords;
+	
+	public InterpolativeStringListPropertyEditor(String id, PropertyDescriptor propertyDescriptor,
+												 IModel<List<String>> model) {
+		super(id, propertyDescriptor, model);
+	}
+
+	@SuppressWarnings("unchecked")
+	private List<String> newList() {
+		if (ClassUtils.isConcrete(getDescriptor().getPropertyClass())) {
+			try {
+				return (List<String>) getDescriptor().getPropertyClass().getDeclaredConstructor().newInstance();
+			} catch (InstantiationException | IllegalAccessException | IllegalArgumentException
+					 | InvocationTargetException | NoSuchMethodException | SecurityException e) {
+				throw new RuntimeException(e);
+			}
+		} else {
+			return new ArrayList<>();
+		}
+	}
+
+	@Override
+	protected void onInitialize() {
+		super.onInitialize();
+
+		List<String> list = getModelObject();
+		if (list == null)
+			list = newList();
+
+		WebMarkupContainer table = new WebMarkupContainer("table") {
+
+			@Override
+			protected void onComponentTag(ComponentTag tag) {
+				super.onComponentTag(tag);
+				if (rows.size() == 0)
+					NoRecordsBehavior.decorate(tag);
+			}
+
+		};
+		add(table);
+		
+		rows = new RepeatingView("elements");
+		table.add(rows);
+
+		for (Serializable element: list)
+			addRow((String) element);
+
+		add(new AjaxButton("addElement") {
+
+			@SuppressWarnings("deprecation")
+			@Override
+			protected void onSubmit(AjaxRequestTarget target, Form<?> form) {
+				super.onSubmit(target, form);
+				markFormDirty(target);
+
+				Component lastRow;
+				if (rows.size() != 0)
+					lastRow = rows.get(rows.size() - 1);
+				else
+					lastRow = null;
+
+				Component newRow = addRow(null);
+				String script = String.format("$('<tr id=\"%s\"></tr>')", newRow.getMarkupId());
+				if (lastRow != null)
+					script += ".insertAfter('#" + lastRow.getMarkupId() + "');";
+				else
+					script += ".appendTo('#" + InterpolativeStringListPropertyEditor.this.getMarkupId() + ">div>table>tbody');";
+
+				target.prependJavaScript(script);
+				target.add(newRow);
+				target.add(noRecords);
+				if (rows.size() == 1) {
+					target.appendJavaScript(String.format("$('#%s>div>table').removeClass('%s');",
+							InterpolativeStringListPropertyEditor.this.getMarkupId(), NoRecordsBehavior.CSS_CLASS));
+				}
+
+				onPropertyUpdating(target);
+			}
+
+		}.setDefaultFormProcessing(false));
+
+		table.add(noRecords = new WebMarkupContainer("noRecords") {
+
+			@Override
+			protected void onConfigure() {
+				super.onConfigure();
+				setVisible(rows.size() == 0);
+			}
+
+		});
+		noRecords.setOutputMarkupPlaceholderTag(true);
+
+		add(new SortBehavior() {
+
+			@SuppressWarnings("deprecation")
+			@Override
+			protected void onSort(AjaxRequestTarget target, SortPosition from, SortPosition to) {
+				markFormDirty(target);
+				
+ 				/*
+				List<Component> children = new ArrayList<>();
+				for (Component child: rows)
+					children.add(child);
+
+				Component fromChild = children.remove(from.getItemIndex());
+				children.add(to.getItemIndex(), fromChild);
+				
+				rows.removeAll();
+				for (Component child: children)
+					rows.add(child);
+				*/
+
+				// Do not use code above as removing components outside of a container and add again 
+				// can cause the fenced feedback panel not functioning properly
+				int fromIndex = from.getItemIndex();
+				int toIndex = to.getItemIndex();
+				if (fromIndex < toIndex) {
+					for (int i=0; i<toIndex-fromIndex; i++)
+						rows.swap(fromIndex+i, fromIndex+i+1);
+				} else {
+					for (int i=0; i<fromIndex-toIndex; i++)
+						rows.swap(fromIndex-i, fromIndex-i-1);
+				}
+				onPropertyUpdating(target);
+			}
+
+		}.sortable("tbody"));
+		
+		add(validatable -> {
+			var index = 0;
+			for (var element: validatable.getValue()) {
+				if (element == null)
+					rows.get(index).error("must not be null");
+				index++;
+			}
+		});
+		
+		setOutputMarkupId(true);
+	}
+
+	@Override
+	public void onEvent(IEvent<?> event) {
+		super.onEvent(event);
+
+		if (event.getPayload() instanceof PropertyUpdating) {
+			event.stop();
+			onPropertyUpdating(((PropertyUpdating)event.getPayload()).getHandler());
+		}
+	}
+
+	private WebMarkupContainer addRow(@Nullable String element) {
+		WebMarkupContainer row = new WebMarkupContainer(rows.newChildId());
+		row.setOutputMarkupId(true);
+		rows.add(row);
+
+		TextField<String> input;
+		row.add(input = new TextField<>("elementEditor", Model.of(element)));
+		input.setType(String.class);
+
+		var interpolative = descriptor.getPropertyGetter().getAnnotation(Interpolative.class);
+		InterpolativeAssistBehavior inputAssist = new InterpolativeAssistBehavior() {
+
+			@SuppressWarnings("unchecked")
+			@Override
+			protected List<InputSuggestion> suggestVariables(String matchWith) {
+				String suggestionMethod = interpolative.variableSuggester();
+				if (suggestionMethod.length() != 0) {
+					return (List<InputSuggestion>) ReflectionUtils.invokeStaticMethod(
+							descriptor.getBeanClass(), suggestionMethod, new Object[] {matchWith});
+				} else {
+					return Lists.newArrayList();
+				}
+			}
+
+			@SuppressWarnings("unchecked")
+			@Override
+			protected List<InputSuggestion> suggestLiterals(String matchWith) {
+				String suggestionMethod = interpolative.literalSuggester();
+				if (suggestionMethod.length() != 0) {
+					return (List<InputSuggestion>) ReflectionUtils.invokeStaticMethod(
+							descriptor.getBeanClass(), suggestionMethod, new Object[] {matchWith});
+				} else {
+					return Lists.newArrayList();
+				}
+			}
+
+		};
+		
+		input.add(inputAssist);
+		input.add(AttributeAppender.append("spellcheck", "false"));
+		input.add(AttributeAppender.append("autocomplete", "off"));
+
+		input.add(new OnTypingDoneBehavior() {
+
+			@Override
+			protected void onTypingDone(AjaxRequestTarget target) {
+				onPropertyUpdating(target);
+			}
+
+		});
+		input.add(newPlaceholderModifier());		
+		
+		row.add(new FencedFeedbackPanel("feedback", row));
+		
+		row.add(new AjaxButton("deleteElement") {
+
+			@Override
+			protected void onSubmit(AjaxRequestTarget target, Form<?> form) {
+				super.onSubmit(target, form);
+				markFormDirty(target);
+				target.appendJavaScript(String.format("$('#%s').remove();", row.getMarkupId()));
+				rows.remove(row);
+				target.add(noRecords);
+
+				if (rows.size() == 0) {
+					target.appendJavaScript(String.format("$('#%s>div>table').addClass('%s');",
+							InterpolativeStringListPropertyEditor.this.getMarkupId(), NoRecordsBehavior.CSS_CLASS));
+				}
+
+				onPropertyUpdating(target);
+			}
+
+		}.setDefaultFormProcessing(false));
+
+		return row;
+	}
+	
+	@Override
+	public void error(PathNode propertyNode, Path pathInProperty, String errorMessage) {
+		int index = ((PathNode.Indexed) propertyNode).getIndex();
+		rows.get(index).error(errorMessage);
+	}
+
+	@Override
+	protected String getInvalidClass() {
+		return null;
+	}
+
+	@Override
+	protected List<String> convertInputToValue() throws ConversionException {
+		List<String> newList = newList();
+
+		for (Component row: rows) {
+			TextField<String> elementEditor = (TextField<String>) row.get("elementEditor");
+			newList.add(elementEditor.getConvertedInput());			
+		}
+		return newList;
+	}
+
+	@Override
+	public boolean needExplicitSubmit() {
+		return true;
+	}
+	
+}

server-core/src/main/java/io/onedev/server/web/editable/interpolativestringlist/InterpolativeStringListPropertyViewer.html

@@ -0,0 +1,3 @@
+<wicket:panel>
+	<wicket:container wicket:id="elements"><span wicket:id="value" class="badge badge-light mr-2"></span></wicket:container>
+</wicket:panel>

server-core/src/main/java/io/onedev/server/web/editable/interpolativestringlist/InterpolativeStringListPropertyViewer.java

@@ -0,0 +1,34 @@
+package io.onedev.server.web.editable.interpolativestringlist;
+
+import io.onedev.server.web.editable.PropertyDescriptor;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.list.ListItem;
+import org.apache.wicket.markup.html.list.ListView;
+import org.apache.wicket.markup.html.panel.Panel;
+
+import java.util.List;
+
+@SuppressWarnings("serial")
+public class InterpolativeStringListPropertyViewer extends Panel {
+	
+	private final List<String> elements;
+	
+	public InterpolativeStringListPropertyViewer(String id, PropertyDescriptor descriptor, List<String> elements) {
+		super(id);
+		this.elements = elements;
+	}
+
+	@Override
+	protected void onInitialize() {
+		super.onInitialize();
+		
+		add(new ListView<>("elements", elements) {
+
+			@Override
+			protected void populateItem(ListItem<String> item) {
+				item.add(new Label("value", item.getModelObject()));
+			}
+		});
+	}
+
+}

server-core/src/main/java/io/onedev/server/web/editable/polymorphic/PolymorphicPropertyEditor.java

@@ -61,7 +61,7 @@ public class PolymorphicPropertyEditor extends PropertyEditor<Serializable> {
 			}
 
 			@Override
-			protected void onTypeChanged(AjaxRequestTarget target) {
+			protected void onTypeChanging(AjaxRequestTarget target) {
 				onPropertyUpdating(target);
 				if (editor.isDefined()) {
 					target.appendJavaScript(String.format("$('#%s').addClass('property-defined');",

server-core/src/main/java/io/onedev/server/web/editable/polymorphiclist/PolymorphicListPropertyEditor.java

@@ -24,8 +24,6 @@ import org.apache.wicket.markup.repeater.RepeatingView;
 import org.apache.wicket.model.IModel;
 import org.apache.wicket.model.Model;
 import org.apache.wicket.util.convert.ConversionException;
-import org.apache.wicket.validation.IValidatable;
-import org.apache.wicket.validation.IValidator;
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
@@ -218,7 +216,7 @@ public class PolymorphicListPropertyEditor extends PropertyEditor<List<Serializa
 			}
 
 			@Override
-			protected void onTypeChanged(AjaxRequestTarget target) {
+			protected void onTypeChanging(AjaxRequestTarget target) {
 				onPropertyUpdating(target);
 			}
 			
@@ -262,12 +260,11 @@ public class PolymorphicListPropertyEditor extends PropertyEditor<List<Serializa
 	@Override
 	public void error(PathNode propertyNode, Path pathInProperty, String errorMessage) {
 		int index = ((PathNode.Indexed) propertyNode).getIndex();
-		String messagePrefix = "Item " + (index+1) + ": ";
 		PathNode.Named named = (PathNode.Named) pathInProperty.takeNode();
 		if (named != null) 
 			getElementEditorAtRow(index).error(named, pathInProperty, errorMessage);
 		else 
-			error(messagePrefix + errorMessage);
+			rows.get(index).error(errorMessage);
 	}
 
 	@Override

server-core/src/main/java/io/onedev/server/web/editable/string/StringPropertyEditor.java

@@ -1,7 +1,10 @@
 package io.onedev.server.web.editable.string;
 
-import java.lang.reflect.Method;
-
+import io.onedev.server.annotation.Multiline;
+import io.onedev.server.web.behavior.OnTypingDoneBehavior;
+import io.onedev.server.web.behavior.inputassist.InputAssistBehavior;
+import io.onedev.server.web.editable.PropertyDescriptor;
+import io.onedev.server.web.editable.PropertyEditor;
 import org.apache.wicket.ajax.AjaxRequestTarget;
 import org.apache.wicket.behavior.AttributeAppender;
 import org.apache.wicket.markup.html.form.FormComponent;
@@ -12,11 +15,7 @@ import org.apache.wicket.model.IModel;
 import org.apache.wicket.model.Model;
 import org.apache.wicket.util.convert.ConversionException;
 
-import io.onedev.server.web.behavior.OnTypingDoneBehavior;
-import io.onedev.server.web.behavior.inputassist.InputAssistBehavior;
-import io.onedev.server.web.editable.PropertyDescriptor;
-import io.onedev.server.web.editable.PropertyEditor;
-import io.onedev.server.annotation.Multiline;
+import java.lang.reflect.Method;
 
 @SuppressWarnings("serial")
 public class StringPropertyEditor extends PropertyEditor<String> {
@@ -32,14 +31,14 @@ public class StringPropertyEditor extends PropertyEditor<String> {
 	@Override
 	protected void onInitialize() {
 		super.onInitialize();
-		
+
 		Method getter = getDescriptor().getPropertyGetter();
 		var multiline = getter.getAnnotation(Multiline.class);
 		if (multiline != null) {
 			Fragment fragment = new Fragment("content", "multiLineFrag", this);
 			fragment.add(input = new TextArea<>("input", Model.of(getModelObject())) {
 
-				@Override
+			@Override
 				protected boolean shouldTrimInput() {
 					return false;
 				}
@@ -56,7 +55,7 @@ public class StringPropertyEditor extends PropertyEditor<String> {
 			fragment.add(input = new TextField<String>("input", Model.of(getModelObject())));
 			input.setType(getDescriptor().getPropertyClass());
 			add(fragment);
-		}
+			}
 		input.setLabel(Model.of(getDescriptor().getDisplayName()));		
 		
 		if (inputAssist != null) {

server-core/src/main/java/io/onedev/server/web/editable/stringlist/StringListEditSupport.java

@@ -1,21 +1,22 @@
 package io.onedev.server.web.editable.stringlist;
 
-import io.onedev.server.util.ReflectionUtils;
 import io.onedev.server.web.editable.*;
 import org.apache.wicket.Component;
-import org.apache.wicket.markup.html.basic.Label;
 import org.apache.wicket.model.IModel;
 
 import java.lang.reflect.AnnotatedElement;
 import java.util.List;
 
+import static io.onedev.server.util.ReflectionUtils.getCollectionElementClass;
+
 @SuppressWarnings("serial")
 public class StringListEditSupport implements EditSupport {
 
 	@Override
 	public PropertyContext<?> getEditContext(PropertyDescriptor descriptor) {
+		var propertyGetter = descriptor.getPropertyGetter();
 		if (List.class.isAssignableFrom(descriptor.getPropertyClass()) 
-				&& ReflectionUtils.getCollectionElementClass(descriptor.getPropertyGetter().getGenericReturnType()) == String.class) {
+				&& getCollectionElementClass(propertyGetter.getGenericReturnType()) == String.class) {
 						
 			return new PropertyContext<List<String>>(descriptor) {
 
@@ -27,14 +28,7 @@ public class StringListEditSupport implements EditSupport {
 						@Override
 						protected Component newContent(String id, PropertyDescriptor propertyDescriptor) {
 							if (model.getObject() != null && !model.getObject().isEmpty()) {
-								String content = "";
-								for (String each: model.getObject()) {
-									if (content.length() == 0)
-										content += each.toString();
-									else
-										content += ", " + each.toString();
-								}
-								return new Label(id, content);
+								return new StringListPropertyViewer(id, propertyDescriptor, model.getObject());
 							} else { 
 								return new EmptyValueLabel(id) {
 

server-core/src/main/java/io/onedev/server/web/editable/stringlist/StringListPropertyViewer.html

@@ -0,0 +1,3 @@
+<wicket:panel>
+	<wicket:container wicket:id="elements"><span wicket:id="value" class="badge badge-light mr-2"></span></wicket:container>
+</wicket:panel>

server-core/src/main/java/io/onedev/server/web/editable/stringlist/StringListPropertyViewer.java

@@ -0,0 +1,34 @@
+package io.onedev.server.web.editable.stringlist;
+
+import io.onedev.server.web.editable.PropertyDescriptor;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.list.ListItem;
+import org.apache.wicket.markup.html.list.ListView;
+import org.apache.wicket.markup.html.panel.Panel;
+
+import java.util.List;
+
+@SuppressWarnings("serial")
+public class StringListPropertyViewer extends Panel {
+	
+	private final List<String> elements;
+	
+	public StringListPropertyViewer(String id, PropertyDescriptor descriptor, List<String> elements) {
+		super(id);
+		this.elements = elements;
+	}
+
+	@Override
+	protected void onInitialize() {
+		super.onInitialize();
+		
+		add(new ListView<>("elements", elements) {
+
+			@Override
+			protected void populateItem(ListItem<String> item) {
+				item.add(new Label("value", item.getModelObject()));
+			}
+		});
+	}
+
+}

server-core/src/main/java/io/onedev/server/web/mapper/BaseUrlMapper.java

@@ -107,10 +107,7 @@ import io.onedev.server.web.page.project.pullrequests.detail.activities.PullRequ
 import io.onedev.server.web.page.project.pullrequests.detail.changes.PullRequestChangesPage;
 import io.onedev.server.web.page.project.pullrequests.detail.codecomments.PullRequestCodeCommentsPage;
 import io.onedev.server.web.page.project.setting.avatar.AvatarEditPage;
-import io.onedev.server.web.page.project.setting.build.BuildPreservationsPage;
-import io.onedev.server.web.page.project.setting.build.DefaultFixedIssueFiltersPage;
-import io.onedev.server.web.page.project.setting.build.JobPropertiesPage;
-import io.onedev.server.web.page.project.setting.build.JobSecretsPage;
+import io.onedev.server.web.page.project.setting.build.*;
 import io.onedev.server.web.page.project.setting.code.analysis.CodeAnalysisSettingPage;
 import io.onedev.server.web.page.project.setting.code.branchprotection.BranchProtectionsPage;
 import io.onedev.server.web.page.project.setting.code.git.GitPackConfigPage;
@@ -381,6 +378,7 @@ public class BaseUrlMapper extends CompoundRequestMapper {
 		add(new ProjectPageMapper("${project}/~settings/build/job-properties", JobPropertiesPage.class));
 		add(new ProjectPageMapper("${project}/~settings/build/build-preserve-rules", BuildPreservationsPage.class));
 		add(new ProjectPageMapper("${project}/~settings/build/default-fixed-issues-filter", DefaultFixedIssueFiltersPage.class));
+		add(new ProjectPageMapper("${project}/~settings/build/cache-management", CacheManagementPage.class));
 		add(new ProjectPageMapper("${project}/~settings/service-desk", ServiceDeskSettingPage.class));
 		add(new ProjectPageMapper("${project}/~settings/web-hooks", WebHooksPage.class));
 		add(new ProjectPageMapper("${project}/~settings/${" + ContributedProjectSettingPage.PARAM_SETTING + "}", 

server-core/src/main/java/io/onedev/server/web/page/admin/databasebackup/DatabaseBackupPage.java

@@ -1,6 +1,7 @@
 package io.onedev.server.web.page.admin.databasebackup;
 
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.ZipUtils;
 import io.onedev.server.OneDev;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.entitymanager.SettingManager;
@@ -77,7 +78,7 @@ public class DatabaseBackupPage extends AdministrationPage {
 						try {
 							DataManager databaseManager = OneDev.getInstance(DataManager.class);
 							databaseManager.exportData(tempDir);
-							FileUtils.zip(tempDir, attributes.getResponse().getOutputStream());
+							ZipUtils.zip(tempDir, attributes.getResponse().getOutputStream());
 						} finally {
 							FileUtils.deleteDir(tempDir);
 						}

server-core/src/main/java/io/onedev/server/web/page/admin/groupmanagement/GroupListPage.java

@@ -1,10 +1,19 @@
 package io.onedev.server.web.page.admin.groupmanagement;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.GroupManager;
+import io.onedev.server.model.Group;
+import io.onedev.server.security.SecurityUtils;
+import io.onedev.server.web.WebConstants;
+import io.onedev.server.web.WebSession;
+import io.onedev.server.web.ajaxlistener.ConfirmClickListener;
+import io.onedev.server.web.behavior.OnTypingDoneBehavior;
+import io.onedev.server.web.component.datatable.DefaultDataTable;
+import io.onedev.server.web.component.link.ActionablePageLink;
+import io.onedev.server.web.page.admin.AdministrationPage;
+import io.onedev.server.web.page.admin.groupmanagement.create.NewGroupPage;
+import io.onedev.server.web.page.admin.groupmanagement.profile.GroupProfilePage;
+import io.onedev.server.web.util.PagingHistorySupport;
 import org.apache.wicket.Component;
 import org.apache.wicket.Session;
 import org.apache.wicket.ajax.AjaxRequestTarget;
@@ -29,20 +38,10 @@ import org.apache.wicket.model.Model;
 import org.apache.wicket.request.cycle.RequestCycle;
 import org.apache.wicket.request.mapper.parameter.PageParameters;
 
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.GroupManager;
-import io.onedev.server.model.Group;
-import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.web.WebConstants;
-import io.onedev.server.web.WebSession;
-import io.onedev.server.web.ajaxlistener.ConfirmClickListener;
-import io.onedev.server.web.behavior.OnTypingDoneBehavior;
-import io.onedev.server.web.component.datatable.DefaultDataTable;
-import io.onedev.server.web.component.link.ActionablePageLink;
-import io.onedev.server.web.page.admin.AdministrationPage;
-import io.onedev.server.web.page.admin.groupmanagement.create.NewGroupPage;
-import io.onedev.server.web.page.admin.groupmanagement.profile.GroupProfilePage;
-import io.onedev.server.web.util.PagingHistorySupport;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 @SuppressWarnings("serial")
 public class GroupListPage extends AdministrationPage {
@@ -132,23 +131,17 @@ public class GroupListPage extends AdministrationPage {
 				setResponsePage(NewGroupPage.class);
 			}
 			
-			@Override
-			protected void onConfigure() {
-				super.onConfigure();
-				setVisible(SecurityUtils.isAdministrator());
-			}
-			
 		});
 		
 		List<IColumn<Group, Void>> columns = new ArrayList<>();
 		
-		columns.add(new AbstractColumn<Group, Void>(Model.of("Name")) {
+		columns.add(new AbstractColumn<>(Model.of("Name")) {
 
 			@Override
 			public void populateItem(Item<ICellPopulator<Group>> cellItem, String componentId, IModel<Group> rowModel) {
 				Fragment fragment = new Fragment(componentId, "nameFrag", GroupListPage.this);
 				Group group = rowModel.getObject();
-				WebMarkupContainer link = new ActionablePageLink("link", 
+				WebMarkupContainer link = new ActionablePageLink("link",
 						GroupProfilePage.class, GroupProfilePage.paramsOf(group)) {
 
 					@Override
@@ -157,7 +150,7 @@ public class GroupListPage extends AdministrationPage {
 								GroupListPage.class, getPageParameters()).toString();
 						WebSession.get().setRedirectUrlAfterDelete(Group.class, redirectUrlAfterDelete);
 					}
-					
+
 				};
 				link.add(new Label("label", group.getName()));
 				fragment.add(link);
@@ -165,14 +158,14 @@ public class GroupListPage extends AdministrationPage {
 			}
 		});
 		
-		columns.add(new AbstractColumn<Group, Void>(Model.of("Is Site Admin")) {
+		columns.add(new AbstractColumn<>(Model.of("Is Site Admin")) {
 
 			@Override
 			public void populateItem(Item<ICellPopulator<Group>> cellItem, String componentId,
-					IModel<Group> rowModel) {
+									 IModel<Group> rowModel) {
 				cellItem.add(new Label(componentId, rowModel.getObject().isAdministrator()));
 			}
-			
+
 		});
 		
 		columns.add(new AbstractColumn<Group, Void>(Model.of("Can Create Root Projects")) {
@@ -186,12 +179,12 @@ public class GroupListPage extends AdministrationPage {
 			
 		});
 		
-		columns.add(new AbstractColumn<Group, Void>(Model.of("")) {
+		columns.add(new AbstractColumn<>(Model.of("")) {
 
 			@Override
 			public void populateItem(Item<ICellPopulator<Group>> cellItem, String componentId, IModel<Group> rowModel) {
 				Fragment fragment = new Fragment(componentId, "actionFrag", GroupListPage.this);
-				
+
 				fragment.add(new AjaxLink<Void>("delete") {
 
 					@Override
@@ -209,14 +202,8 @@ public class GroupListPage extends AdministrationPage {
 						target.add(groupsTable);
 					}
 
-					@Override
-					protected void onConfigure() {
-						super.onConfigure();
-						setVisible(SecurityUtils.isAdministrator());
-					}
-					
 				});
-				
+
 				cellItem.add(fragment);
 			}
 
@@ -224,14 +211,14 @@ public class GroupListPage extends AdministrationPage {
 			public String getCssClass() {
 				return "actions";
 			}
-			
+
 		});
 		
-		SortableDataProvider<Group, Void> dataProvider = new SortableDataProvider<Group, Void>() {
+		SortableDataProvider<Group, Void> dataProvider = new SortableDataProvider<>() {
 
 			@Override
 			public Iterator<? extends Group> iterator(long first, long count) {
-				return OneDev.getInstance(GroupManager.class).query(query, (int)first, (int)count).iterator();
+				return OneDev.getInstance(GroupManager.class).query(query, (int) first, (int) count).iterator();
 			}
 
 			@Override
@@ -242,13 +229,13 @@ public class GroupListPage extends AdministrationPage {
 			@Override
 			public IModel<Group> model(Group object) {
 				Long id = object.getId();
-				return new LoadableDetachableModel<Group>() {
+				return new LoadableDetachableModel<>() {
 
 					@Override
 					protected Group load() {
 						return OneDev.getInstance(GroupManager.class).load(id);
 					}
-					
+
 				};
 			}
 		};

server-core/src/main/java/io/onedev/server/web/page/admin/usermanagement/UserListPage.java

@@ -8,8 +8,6 @@ import io.onedev.server.model.EmailAddress;
 import io.onedev.server.model.User;
 import io.onedev.server.security.SecurityUtils;
 import io.onedev.server.util.Similarities;
-import io.onedev.server.util.facade.EmailAddressCache;
-import io.onedev.server.util.facade.UserCache;
 import io.onedev.server.web.WebConstants;
 import io.onedev.server.web.WebSession;
 import io.onedev.server.web.behavior.OnTypingDoneBehavior;
@@ -179,12 +177,6 @@ public class UserListPage extends AdministrationPage {
 			public void onClick() {
 				setResponsePage(NewUserPage.class);
 			}
-
-			@Override
-			protected void onConfigure() {
-				super.onConfigure();
-				setVisible(SecurityUtils.isAdministrator());
-			}
 			
 		});
 

server-core/src/main/java/io/onedev/server/web/page/project/ProjectPage.java

@@ -57,10 +57,7 @@ import io.onedev.server.web.page.project.setting.ProjectSettingPage;
 import io.onedev.server.web.page.project.setting.authorization.GroupAuthorizationsPage;
 import io.onedev.server.web.page.project.setting.authorization.UserAuthorizationsPage;
 import io.onedev.server.web.page.project.setting.avatar.AvatarEditPage;
-import io.onedev.server.web.page.project.setting.build.BuildPreservationsPage;
-import io.onedev.server.web.page.project.setting.build.DefaultFixedIssueFiltersPage;
-import io.onedev.server.web.page.project.setting.build.JobPropertiesPage;
-import io.onedev.server.web.page.project.setting.build.JobSecretsPage;
+import io.onedev.server.web.page.project.setting.build.*;
 import io.onedev.server.web.page.project.setting.code.analysis.CodeAnalysisSettingPage;
 import io.onedev.server.web.page.project.setting.code.branchprotection.BranchProtectionsPage;
 import io.onedev.server.web.page.project.setting.code.git.GitPackConfigPage;
@@ -302,6 +299,8 @@ public abstract class ProjectPage extends LayoutPage implements ProjectAware {
 					BuildPreservationsPage.class, BuildPreservationsPage.paramsOf(getProject())));
 			buildSettingMenuItems.add(new SidebarMenuItem.Page(null, "Default Fixed Issue Filters", 
 					DefaultFixedIssueFiltersPage.class, DefaultFixedIssueFiltersPage.paramsOf(getProject())));
+			buildSettingMenuItems.add(new SidebarMenuItem.Page(null, "Cache Management",
+					CacheManagementPage.class, CacheManagementPage.paramsOf(getProject())));
 			
 			settingMenuItems.add(new SidebarMenuItem.SubMenu(null, "Build", buildSettingMenuItems));
 			

server-core/src/main/java/io/onedev/server/web/page/project/setting/build/CacheManagementPage.html

@@ -0,0 +1,16 @@
+<wicket:extend>
+	<div class="card">
+		<div class="card-body">
+			<h6 class="mb-4">Cache Preserve Days</h6>
+			<form wicket:id="cacheSetting" class="mb-5">
+				<div wicket:id="editor"></div>
+				<input type="submit" class="btn btn-primary" value="Update">
+			</form>
+			<h6 class="border-top pt-4 mb-2">Caches</h6>
+			<table wicket:id="caches" class="table table-hover mb-0"></table>
+		</div>
+	</div>
+	<wicket:fragment wicket:id="actionFrag">
+		<a wicket:id="delete"><wicket:svg href="trash" class="icon"/></a>
+	</wicket:fragment>
+</wicket:extend>

server-core/src/main/java/io/onedev/server/web/page/project/setting/build/CacheManagementPage.java

@@ -0,0 +1,196 @@
+package io.onedev.server.web.page.project.setting.build;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.JobCacheManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.model.JobCache;
+import io.onedev.server.persistence.dao.EntityCriteria;
+import io.onedev.server.util.DateUtils;
+import io.onedev.server.web.WebConstants;
+import io.onedev.server.web.component.datatable.DefaultDataTable;
+import io.onedev.server.web.editable.BeanContext;
+import io.onedev.server.web.util.PagingHistorySupport;
+import org.apache.commons.io.FileUtils;
+import org.apache.wicket.Component;
+import org.apache.wicket.Session;
+import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.ajax.markup.html.AjaxLink;
+import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;
+import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;
+import org.apache.wicket.extensions.markup.html.repeater.data.table.DataTable;
+import org.apache.wicket.extensions.markup.html.repeater.data.table.IColumn;
+import org.apache.wicket.extensions.markup.html.repeater.util.SortableDataProvider;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.markup.html.panel.Fragment;
+import org.apache.wicket.markup.repeater.Item;
+import org.apache.wicket.model.IModel;
+import org.apache.wicket.model.LoadableDetachableModel;
+import org.apache.wicket.model.Model;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+import org.hibernate.criterion.Order;
+import org.hibernate.criterion.Restrictions;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static io.onedev.server.model.JobCache.PROP_PROJECT;
+
+public class CacheManagementPage extends ProjectBuildSettingPage {
+	
+	private static final String PARAM_PAGE = "page";
+	
+	private DataTable<JobCache, Void> cachesTable;
+	
+	public CacheManagementPage(PageParameters params) {
+		super(params);
+	}
+
+	@Override
+	protected void onInitialize() {
+		super.onInitialize();
+		
+		var bean = new CacheSettingBean();
+		bean.setPreserveDays(getProject().getBuildSetting().getCachePreserveDays());
+		var form = new Form<Void>("cacheSetting") {
+			@Override
+			protected void onSubmit() {
+				super.onSubmit();
+				getProject().getBuildSetting().setCachePreserveDays(bean.getPreserveDays());
+				OneDev.getInstance(ProjectManager.class).update(getProject());
+			}
+			
+		};
+		form.add(BeanContext.edit("editor", bean));
+		add(form);
+
+		List<IColumn<JobCache, Void>> columns = new ArrayList<>();
+
+		columns.add(new AbstractColumn<>(Model.of("Key")) {
+
+			@Override
+			public void populateItem(Item<ICellPopulator<JobCache>> cellItem, String componentId, IModel<JobCache> rowModel) {
+				cellItem.add(new Label(componentId, rowModel.getObject().getKey()));
+			}
+		});
+		
+		columns.add(new AbstractColumn<>(Model.of("Size")) {
+
+			@Override
+			public void populateItem(Item<ICellPopulator<JobCache>> cellItem, String componentId,
+									 IModel<JobCache> rowModel) {
+				var cache = rowModel.getObject();
+				var cacheSize = getCacheManager().getCacheSize(cache.getProject().getId(), cache.getId());
+				if (cacheSize != null)
+					cellItem.add(new Label(componentId, FileUtils.byteCountToDisplaySize(cacheSize)));
+				else 
+					cellItem.add(new Label(componentId, "<i class='text-danger'>File missing or obsolete</i>").setEscapeModelStrings(false));
+			}
+
+		});
+
+		columns.add(new AbstractColumn<>(Model.of("Last Accessed")) {
+
+			@Override
+			public void populateItem(Item<ICellPopulator<JobCache>> cellItem, String componentId,
+									 IModel<JobCache> rowModel) {
+				var cache = rowModel.getObject();
+				cellItem.add(new Label(componentId, DateUtils.formatDate(cache.getAccessDate())));
+			}
+
+		});
+
+		columns.add(new AbstractColumn<>(Model.of("")) {
+
+			@Override
+			public void populateItem(Item<ICellPopulator<JobCache>> cellItem, String componentId, IModel<JobCache> rowModel) {
+				Fragment fragment = new Fragment(componentId, "actionFrag", CacheManagementPage.this);
+
+				fragment.add(new AjaxLink<Void>("delete") {
+
+					@Override
+					public void onClick(AjaxRequestTarget target) {
+						var cache = rowModel.getObject();
+						getCacheManager().delete(cache);
+						Session.get().success("Cache '" + cache.getKey() + "' deleted");
+						target.add(cachesTable);
+					}
+					
+				});
+
+				cellItem.add(fragment);
+			}
+
+			@Override
+			public String getCssClass() {
+				return "actions";
+			}
+
+		});
+
+		SortableDataProvider<JobCache, Void> dataProvider = new SortableDataProvider<>() {
+
+			private EntityCriteria<JobCache> newCriteria() {
+				var criteria = EntityCriteria.of(JobCache.class);
+				criteria.add(Restrictions.eq(PROP_PROJECT, getProject()));
+				return criteria;
+			}
+			
+			@Override
+			public Iterator<? extends JobCache> iterator(long first, long count) {
+				var criteria = newCriteria();
+				criteria.addOrder(Order.desc(JobCache.PROP_ACCESS_DATE));
+				return getCacheManager().query(criteria, (int) first, (int) count).iterator();
+			}
+
+			@Override
+			public long size() {
+				return getCacheManager().count(newCriteria());
+			}
+
+			@Override
+			public IModel<JobCache> model(JobCache object) {
+				Long id = object.getId();
+				return new LoadableDetachableModel<>() {
+
+					@Override
+					protected JobCache load() {
+						return getCacheManager().load(id);
+					}
+
+				};
+			}
+
+		};
+
+		PagingHistorySupport pagingHistorySupport = new PagingHistorySupport() {
+
+			@Override
+			public PageParameters newPageParameters(int currentPage) {
+				PageParameters params = new PageParameters();
+				params.add(PARAM_PAGE, currentPage+1);
+				return params;
+			}
+
+			@Override
+			public int getCurrentPage() {
+				return getPageParameters().get(PARAM_PAGE).toInt(1)-1;
+			}
+
+		};
+
+		add(cachesTable = new DefaultDataTable<>("caches", columns, dataProvider,
+				WebConstants.PAGE_SIZE, pagingHistorySupport));
+	}
+
+	@Override
+	protected Component newProjectTitle(String componentId) {
+		return new Label(componentId, "Job Cache Management");
+	}
+	
+	private JobCacheManager getCacheManager() {
+		return OneDev.getInstance(JobCacheManager.class);
+	}
+	
+}

server-core/src/main/java/io/onedev/server/web/page/project/setting/build/CacheSettingBean.java

@@ -0,0 +1,27 @@
+package io.onedev.server.web.page.project.setting.build;
+
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.OmitName;
+import io.onedev.server.model.support.build.ProjectBuildSetting;
+
+import javax.validation.constraints.Min;
+import java.io.Serializable;
+
+@Editable
+public class CacheSettingBean implements Serializable {
+	
+	private Integer preserveDays;
+
+	@Editable(placeholder = "Inherit from parent", rootPlaceholder = ProjectBuildSetting.DEFAULT_CACHE_PRESERVE_DAYS + " days", 
+			description = "Cache will be deleted to save space if not accessed for this number of days")
+	@OmitName
+	@Min(1)
+	public Integer getPreserveDays() {
+		return preserveDays;
+	}
+
+	public void setPreserveDays(Integer preserveDays) {
+		this.preserveDays = preserveDays;
+	}
+	
+}

server-core/src/main/java/io/onedev/server/web/page/test/TestPage.java

@@ -1,5 +1,6 @@
 package io.onedev.server.web.page.test;
 
+import io.onedev.k8shelper.KubernetesHelper;
 import io.onedev.server.web.page.base.BasePage;
 import org.apache.wicket.markup.head.IHeaderResponse;
 import org.apache.wicket.markup.head.JavaScriptHeaderItem;

server-core/src/main/java/io/onedev/server/web/resource/AgentLibResource.java

@@ -7,6 +7,7 @@ import java.util.Collection;
 
 import javax.servlet.http.HttpServletRequest;
 
+import io.onedev.commons.utils.TarUtils;
 import org.apache.tika.mime.MimeTypes;
 import org.apache.wicket.request.resource.AbstractResource;
 
@@ -51,7 +52,7 @@ public class AgentLibResource extends AbstractResource {
 						}
 						
 						OutputStream os = attributes.getResponse().getOutputStream();
-						FileUtils.tar(tempDir, os, false);
+						TarUtils.tar(tempDir, os, false);
 					} finally {
 						FileUtils.deleteDir(tempDir);
 					}

server-core/src/main/java/io/onedev/server/web/resource/AgentResource.java

@@ -3,9 +3,7 @@ package io.onedev.server.web.resource;
 import com.google.common.collect.Sets;
 import io.onedev.agent.Agent;
 import io.onedev.commons.bootstrap.Bootstrap;
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.FileUtils;
-import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.*;
 import io.onedev.server.OneDev;
 import io.onedev.server.entitymanager.AgentManager;
 import io.onedev.server.entitymanager.AgentTokenManager;
@@ -101,10 +99,10 @@ public class AgentResource extends AbstractResource {
 
 					if (fileName.endsWith("zip")) {
 						File packageFile = new File(tempDir, fileName);
-						FileUtils.zip(agentDir, packageFile, "agent/boot/wrapper-*, agent/bin/*.sh");
+						ZipUtils.zip(agentDir, packageFile, "agent/boot/wrapper-*, agent/bin/*.sh");
 						IOUtils.copy(packageFile, attributes.getResponse().getOutputStream());
 					} else {
-						FileUtils.tar(agentDir, Sets.newHashSet("**"), Sets.newHashSet(), 
+						TarUtils.tar(agentDir, Sets.newHashSet("**"), Sets.newHashSet(), 
 								Sets.newHashSet("agent/boot/wrapper-*", "agent/bin/*.sh"),
 								attributes.getResponse().getOutputStream(), true);
 					}

server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java

@@ -7,6 +7,7 @@ import io.onedev.commons.loader.ManagedSerializedForm;
 import io.onedev.commons.utils.FileUtils;
 import io.onedev.commons.utils.PathUtils;
 import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TarUtils;
 import io.onedev.k8shelper.KubernetesHelper;
 import io.onedev.server.OneDev;
 import io.onedev.server.cluster.ClusterManager;
@@ -1172,7 +1173,7 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager
 						KubernetesHelper.BEARER + " " + clusterManager.getCredential());
 				try (Response response = builder.get()) {
 					KubernetesHelper.checkStatus(response);
-					FileUtils.untar(
+					TarUtils.untar(
 							response.readEntity(InputStream.class),
 							getEnvDir(targetProjectId.toString()), false);
 				}

server-core/src/main/java/io/onedev/server/xodus/DefaultVisitInfoManager.java

@@ -3,6 +3,7 @@ package io.onedev.server.xodus;
 import com.google.common.collect.Lists;
 import io.onedev.commons.loader.ManagedSerializedForm;
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.TarUtils;
 import io.onedev.k8shelper.KubernetesHelper;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.entitymanager.ProjectManager;
@@ -249,7 +250,7 @@ public class DefaultVisitInfoManager extends AbstractMultiEnvironmentManager
 						KubernetesHelper.BEARER + " " + clusterManager.getCredential());
 				try (Response response = builder.get()) {
 					KubernetesHelper.checkStatus(response);
-					FileUtils.untar(
+					TarUtils.untar(
 							response.readEntity(InputStream.class),
 							getEnvDir(projectId.toString()), false);
 				}

server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java

@@ -10,6 +10,7 @@ import java.io.InputStream;
 import java.util.HashSet;
 import java.util.Set;
 
+import io.onedev.commons.utils.ZipUtils;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
@@ -327,7 +328,7 @@ public class GitUtilsTest extends AbstractGitTest {
 		
 		tempDir = FileUtils.createTempDir();
 		try (InputStream is = Resources.getResource(GitUtilsTest.class, "git-conflict-link-link.zip").openStream()) {
-			FileUtils.unzip(is, tempDir);
+			ZipUtils.unzip(is, tempDir);
 			try (Git git = Git.open(tempDir)) {
 				ObjectId mergeCommitId;
 
@@ -362,7 +363,7 @@ public class GitUtilsTest extends AbstractGitTest {
 		
 		tempDir = FileUtils.createTempDir();
 		try (InputStream is = Resources.getResource(GitUtilsTest.class, "git-conflict-link-file.zip").openStream()) {
-			FileUtils.unzip(is, tempDir);
+			ZipUtils.unzip(is, tempDir);
 			try (Git git = Git.open(tempDir)) {
 				ObjectId mergeCommitId;
 
@@ -411,7 +412,7 @@ public class GitUtilsTest extends AbstractGitTest {
 		
 		tempDir = FileUtils.createTempDir();
 		try (InputStream is = Resources.getResource(GitUtilsTest.class, "git-conflict-link-dir.zip").openStream()) {
-			FileUtils.unzip(is, tempDir);
+			ZipUtils.unzip(is, tempDir);
 			try (Git git = Git.open(tempDir)) {
 				ObjectId mergeCommitId;
 

server-core/src/test/java/io/onedev/server/git/signatureverification/ssh/SignatureVerifierTest.java

@@ -3,6 +3,7 @@ package io.onedev.server.git.signatureverification.ssh;
 import com.google.common.collect.Sets;
 import com.google.common.io.Resources;
 import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.ZipUtils;
 import io.onedev.server.entitymanager.EmailAddressManager;
 import io.onedev.server.entitymanager.GpgKeyManager;
 import io.onedev.server.entitymanager.SettingManager;
@@ -38,7 +39,7 @@ public class SignatureVerifierTest {
     public void verify() {
 		var tempDir = FileUtils.createTempDir();
 		try (InputStream is = Resources.getResource(SignatureVerifierTest.class, "git-signature.zip").openStream()) {
-			FileUtils.unzip(is, tempDir);
+			ZipUtils.unzip(is, tempDir);
 			try (Git git = Git.open(tempDir)) {
 				var emailAddressValue = "foo@example.com";
 				var owner = new User();

server-ee

@@ -1 +1 @@
-Subproject commit b951c0beb6a68c1a3892eca2c55be6f5cd6f4aef
+Subproject commit e47483a7bcb33b758787c1ac99ea407aef02c45d

server-plugin/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-archetype/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<build>
 		<resources>

server-plugin/server-plugin-authenticator-ldap/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<properties>
 		<moduleClass>io.onedev.server.plugin.authenticator.ldap.LdapModule</moduleClass>

server-plugin/server-plugin-buildspec-dotnet/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java

@@ -1,39 +1,27 @@
 package io.onedev.server.plugin.buildspec.dotnet;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
 import io.onedev.k8shelper.ExecuteCondition;
-import io.onedev.server.OneDev;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.buildspec.job.Job;
 import io.onedev.server.buildspec.job.JobSuggestion;
 import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
 import io.onedev.server.buildspec.job.trigger.PullRequestUpdateTrigger;
 import io.onedev.server.buildspec.step.CheckoutStep;
 import io.onedev.server.buildspec.step.CommandStep;
-import io.onedev.server.git.Blob;
+import io.onedev.server.buildspec.step.GenerateChecksumStep;
+import io.onedev.server.buildspec.step.SetupCacheStep;
 import io.onedev.server.git.BlobIdent;
 import io.onedev.server.git.BlobIdentFilter;
-import io.onedev.server.model.Build;
 import io.onedev.server.model.Project;
-import io.onedev.server.model.support.administration.GroovyScript;
 import io.onedev.server.plugin.report.cobertura.PublishCoberturaReportStep;
 import io.onedev.server.plugin.report.roslynator.PublishRoslynatorReportStep;
 import io.onedev.server.plugin.report.trx.PublishTRXReportStep;
-import io.onedev.server.util.interpolative.VariableInterpolator;
 import org.eclipse.jgit.lib.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 
 public class DotnetJobSuggestion implements JobSuggestion {
-
-	private static final Logger logger = LoggerFactory.getLogger(DotnetJobSuggestion.class);
-	
-	public static final String DETERMINE_DOCKER_IMAGE = "dotnet:determine-docker-image";
 	
 	@Override
 	public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
@@ -46,14 +34,25 @@ public class DotnetJobSuggestion implements JobSuggestion {
 			job.setName("dotnet ci");
 
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
-			
-			String imageName = "@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_DOCKER_IMAGE + "@";
+
+			var generateChecksum = new GenerateChecksumStep();
+			generateChecksum.setName("generate project checksum");
+			generateChecksum.setFiles("**/*.csproj");
+			generateChecksum.setTargetFile("checksum");
+			job.getSteps().add(generateChecksum);
+
+			var setupCache = new SetupCacheStep();
+			setupCache.setName("set up package cache");
+			setupCache.setKey("nuget_packages_@file:checksum@");
+			setupCache.setPath("/root/.nuget/packages");
+			setupCache.getLoadKeys().add("nuget_packages");
+			job.getSteps().add(setupCache);
 			
 			var runTest = new CommandStep();
 			runTest.setName("run tests");
-			runTest.setImage(imageName);
+			runTest.setImage("mcr.microsoft.com/dotnet/sdk");
 			runTest.getInterpreter().setCommands(Lists.newArrayList(
 					"dotnet tool install -g roslynator.dotnet.cli",
 					"dotnet test -l trx --collect:\"XPlat Code Coverage\"", 
@@ -85,35 +84,9 @@ public class DotnetJobSuggestion implements JobSuggestion {
 			job.getTriggers().add(new BranchUpdateTrigger());
 			job.getTriggers().add(new PullRequestUpdateTrigger());
 
-			CacheSpec cache = new CacheSpec();
-			cache.setKey("nuget-cache");
-			cache.setPath("/root/.nuget/packages");
-			job.getCaches().add(cache);
-			
 			jobs.add(job);
 		} 
 		return jobs;
 	}
 
-	public static String determineDockerImage() {
-		var version = "latest";
-		Build build = Build.get();
-		if (build != null) {
-			Project project = build.getProject();
-			ObjectId commitId = build.getCommitId();
-
-			Blob blob = project.getBlob(new BlobIdent(commitId.name(), "global.json"), false);
-			if (blob != null) {
-				ObjectMapper objectMapper = OneDev.getInstance(ObjectMapper.class);
-				try {
-					var node = objectMapper.readTree(blob.getText().getContent());
-					if (node.has("sdk") && node.get("sdk").has("version"))
-						version = node.get("sdk").get("version").asText();
-				} catch (IOException e) {
-					logger.error("Error parsing global.json", e);
-				}
-			}
-		}
-		return "mcr.microsoft.com/dotnet/sdk:" + version;
-	}
 }

server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetModule.java

@@ -1,11 +1,7 @@
 package io.onedev.server.plugin.buildspec.dotnet;
 
-import com.google.common.collect.Lists;
-
 import io.onedev.commons.loader.AbstractPluginModule;
 import io.onedev.server.buildspec.job.JobSuggestion;
-import io.onedev.server.model.support.administration.GroovyScript;
-import io.onedev.server.util.ScriptContribution;
 
 /**
  * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@@ -19,19 +15,6 @@ public class DotnetModule extends AbstractPluginModule {
 		
 		// put your guice bindings here
 		contribute(JobSuggestion.class, DotnetJobSuggestion.class);
-		
-		contribute(ScriptContribution.class, new ScriptContribution() {
-
-			@Override
-			public GroovyScript getScript() {
-				GroovyScript script = new GroovyScript();
-				script.setName(DotnetJobSuggestion.DETERMINE_DOCKER_IMAGE);
-				script.setContent(Lists.newArrayList("io.onedev.server.plugin.buildspec.dotnet.DotnetJobSuggestion.determineDockerImage()"));
-				return script;
-			}
-			
-		});
-		
 	}
 
 }

server-plugin/server-plugin-buildspec-gradle/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleJobSuggestion.java

@@ -2,31 +2,21 @@ package io.onedev.server.plugin.buildspec.gradle;
 
 import com.google.common.collect.Lists;
 import io.onedev.k8shelper.ExecuteCondition;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.buildspec.job.Job;
 import io.onedev.server.buildspec.job.JobSuggestion;
 import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
 import io.onedev.server.buildspec.job.trigger.PullRequestUpdateTrigger;
-import io.onedev.server.buildspec.step.CheckoutStep;
-import io.onedev.server.buildspec.step.CommandStep;
-import io.onedev.server.buildspec.step.SetBuildVersionStep;
+import io.onedev.server.buildspec.step.*;
 import io.onedev.server.git.Blob;
 import io.onedev.server.git.BlobIdent;
-import io.onedev.server.model.Build;
 import io.onedev.server.model.Project;
-import io.onedev.server.model.support.administration.GroovyScript;
 import io.onedev.server.plugin.report.junit.PublishJUnitReportStep;
-import io.onedev.server.util.interpolative.VariableInterpolator;
-import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jgit.lib.ObjectId;
 
-import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Collection;
 
 public class GradleJobSuggestion implements JobSuggestion {
-		
-	public static final String DETERMINE_DOCKER_IMAGE = "gradle:determine-docker-image";
 	
 	@Override
 	public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
@@ -40,10 +30,23 @@ public class GradleJobSuggestion implements JobSuggestion {
 			job.setName("gradle ci");
 			
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
+
+			var generateChecksum = new GenerateChecksumStep();
+			generateChecksum.setName("generate gradle checksum");
+			generateChecksum.setFiles("**/build.gradle **/build.gradle.kts");
+			generateChecksum.setTargetFile("checksum");
+			job.getSteps().add(generateChecksum);
+
+			var setupCache = new SetupCacheStep();
+			setupCache.setName("set up gradle cache");
+			setupCache.setKey("gradle_@file:checksum@");
+			setupCache.setPath("/home/gradle/.gradle/cache");
+			setupCache.getLoadKeys().add("gradle");
+			job.getSteps().add(setupCache);
 			
-			String imageName = "@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_DOCKER_IMAGE + "@";
+			String imageName = "gradle";
 			
 			CommandStep detectBuildVersion = new CommandStep();
 			detectBuildVersion.setName("detect build version");
@@ -74,11 +77,6 @@ public class GradleJobSuggestion implements JobSuggestion {
 			job.getTriggers().add(new BranchUpdateTrigger());
 			job.getTriggers().add(new PullRequestUpdateTrigger());
 
-			CacheSpec cache = new CacheSpec();
-			cache.setKey("gradle-cache");
-			cache.setPath("/home/gradle/.gradle");
-			job.getCaches().add(cache);
-			
 			jobs.add(job);
 		} 
 		
@@ -99,56 +97,4 @@ public class GradleJobSuggestion implements JobSuggestion {
 		return blob;
 	}
 	
-	private static Blob getGradlePropertiesBlob(Project project, ObjectId commitId) {
-		return project.getBlob(new BlobIdent(commitId.name(), "gradle.properties"), false);		
-	}
-	
-	@Nullable
-	private static String getJdkVersion(Blob blob) {
-		for(String line: blob.getText().getContent().split("\n")) {
-			line = line.toLowerCase().trim();
-			if (line.contains("sourcecompatibility") && line.contains("=")) {
-				String jdkVersion = StringUtils.substringAfter(line, "=").trim();
-				jdkVersion = StringUtils.strip(jdkVersion, "'\"").trim();
-				if (jdkVersion.startsWith("JavaVersion.VERSION_"))
-					jdkVersion = jdkVersion.substring("JavaVersion.VERSION_".length());;
-				return jdkVersion;
-			}
-		}
-		return null;
-	}
-
-	@Nullable
-	public static String determineDockerImage() {
-		Build build = Build.get();
-		if (build != null) {
-			String jdkVersion = null;
-			Blob blob = getGradlePropertiesBlob(build.getProject(), build.getCommitId());
-			if (blob != null)
-				jdkVersion = getJdkVersion(blob);
-			if (jdkVersion == null) {
-				blob = getGradleBlob(build.getProject(), build.getCommitId());
-				if (blob != null) 
-					jdkVersion = getJdkVersion(blob);
-			}
-			if (jdkVersion == null) {
-				blob = getKotlinGradleBlob(build.getProject(), build.getCommitId());
-				if (blob != null)
-					jdkVersion = getJdkVersion(blob);
-			}
-			if (jdkVersion != null) {
-				try {
-					if (Integer.parseInt(jdkVersion) > 8)
-						return "gradle";
-				} catch (NumberFormatException e) {
-				}
-				return "gradle:5.6.3-jdk8";
-			} else {
-				return "gradle";
-			}
-		} else {
-			return null;
-		}
-	}
-	
 }

server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleModule.java

@@ -1,11 +1,7 @@
 package io.onedev.server.plugin.buildspec.gradle;
 
-import com.google.common.collect.Lists;
-
 import io.onedev.commons.loader.AbstractPluginModule;
 import io.onedev.server.buildspec.job.JobSuggestion;
-import io.onedev.server.model.support.administration.GroovyScript;
-import io.onedev.server.util.ScriptContribution;
 
 /**
  * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@@ -19,19 +15,6 @@ public class GradleModule extends AbstractPluginModule {
 		
 		// put your guice bindings here
 		contribute(JobSuggestion.class, GradleJobSuggestion.class);
-		
-		contribute(ScriptContribution.class, new ScriptContribution() {
-
-			@Override
-			public GroovyScript getScript() {
-				GroovyScript script = new GroovyScript();
-				script.setName(GradleJobSuggestion.DETERMINE_DOCKER_IMAGE);
-				script.setContent(Lists.newArrayList("io.onedev.server.plugin.buildspec.gradle.GradleJobSuggestion.determineDockerImage()"));
-				return script;
-			}
-			
-		});
-		
 	}
 
 }

server-plugin/server-plugin-buildspec-maven/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenJobSuggestion.java

@@ -2,32 +2,20 @@ package io.onedev.server.plugin.buildspec.maven;
 
 import com.google.common.collect.Lists;
 import io.onedev.k8shelper.ExecuteCondition;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.buildspec.job.Job;
 import io.onedev.server.buildspec.job.JobSuggestion;
 import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
 import io.onedev.server.buildspec.job.trigger.PullRequestUpdateTrigger;
-import io.onedev.server.buildspec.step.CheckoutStep;
-import io.onedev.server.buildspec.step.CommandStep;
-import io.onedev.server.buildspec.step.SetBuildVersionStep;
+import io.onedev.server.buildspec.step.*;
 import io.onedev.server.git.Blob;
 import io.onedev.server.git.BlobIdent;
-import io.onedev.server.model.Build;
 import io.onedev.server.model.Project;
-import io.onedev.server.model.support.administration.GroovyScript;
 import io.onedev.server.plugin.report.junit.PublishJUnitReportStep;
-import io.onedev.server.util.interpolative.VariableInterpolator;
-import org.dom4j.Document;
-import org.dom4j.DocumentException;
-import org.dom4j.Node;
-import org.dom4j.io.SAXReader;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collection;
 
@@ -35,8 +23,6 @@ public class MavenJobSuggestion implements JobSuggestion {
 
 	private static final Logger logger = LoggerFactory.getLogger(MavenJobSuggestion.class);
 	
-	public static final String DETERMINE_DOCKER_IMAGE = "maven:determine-docker-image";
-	
 	@Override
 	public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
 		Collection<Job> jobs = new ArrayList<>();
@@ -47,14 +33,25 @@ public class MavenJobSuggestion implements JobSuggestion {
 			job.setName("maven ci");
 
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
 			
-			String imageName = "@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_DOCKER_IMAGE + "@";
+			var generateChecksum = new GenerateChecksumStep();
+			generateChecksum.setName("generate pom checksum");
+			generateChecksum.setFiles("**/pom.xml");
+			generateChecksum.setTargetFile("checksum");
+			job.getSteps().add(generateChecksum);
+
+			var setupCache = new SetupCacheStep();
+			setupCache.setName("set up repository cache");
+			setupCache.setKey("maven_repository_@file:checksum@");
+			setupCache.setPath("/root/.m2/repository");
+			setupCache.getLoadKeys().add("maven_repository");
+			job.getSteps().add(setupCache);
 			
 			CommandStep detectBuildVersion = new CommandStep();
 			detectBuildVersion.setName("detect build version");
-			detectBuildVersion.setImage(imageName);
+			detectBuildVersion.setImage("maven");
 			detectBuildVersion.getInterpreter().setCommands(Lists.newArrayList(
 					"echo \"Detecting project version (may require some time while downloading maven dependencies)...\"",
 					"echo $(mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout) > buildVersion"));
@@ -67,7 +64,7 @@ public class MavenJobSuggestion implements JobSuggestion {
 			
 			CommandStep runTests = new CommandStep();
 			runTests.setName("run tests");
-			runTests.setImage(imageName);
+			runTests.setImage("maven");
 			runTests.getInterpreter().setCommands(Lists.newArrayList("mvn clean test"));
 			job.getSteps().add(runTests);
 
@@ -81,64 +78,9 @@ public class MavenJobSuggestion implements JobSuggestion {
 			job.getTriggers().add(new BranchUpdateTrigger());
 			job.getTriggers().add(new PullRequestUpdateTrigger());
 			
-			CacheSpec cache = new CacheSpec();
-			cache.setKey("maven-cache");
-			cache.setPath("/root/.m2/repository");
-			job.getCaches().add(cache);
-			
 			jobs.add(job);
 		} 
 		return jobs;
 	}
-
-	@Nullable
-	public static String determineDockerImage() {
-		Build build = Build.get();
-		if (build != null) {
-			Project project = build.getProject();
-			ObjectId commitId = build.getCommitId();
 	
-			Blob blob = project.getBlob(new BlobIdent(commitId.name(), "pom.xml", FileMode.TYPE_FILE), false);
-	
-			Document document;
-			try {
-				document = new SAXReader().read(new StringReader(blob.getText().getContent()));
-			} catch (DocumentException e) {
-				logger.debug("Error parsing pom.xml (project: {}, commit: {})",
-						project.getPath(), commitId.getName(), e);
-				return null;
-			}
-	
-			String javaVersion = "1.8";
-	
-			// Use XPath with localname as POM project element may contain xmlns definition
-			Node node = document.selectSingleNode("//*[local-name()='maven.compiler.source']");
-			if (node != null) {
-				javaVersion = node.getText().trim();
-			} else {
-				node = document.selectSingleNode("//*[local-name()='artifactId' and text()='maven-compiler-plugin']");
-				if (node != null)
-					node = node.getParent().selectSingleNode(".//*[local-name()='source']");
-				if (node != null) {
-					javaVersion = node.getText().trim();
-				} else {
-					// detect java version from Spring initializer generated projects
-					node = document.selectSingleNode("//*[local-name()='java.version']");
-					if (node != null)
-						javaVersion = node.getText().trim();
-				}
-			}
-	
-			try {
-				if (Integer.parseInt(javaVersion) <= 8)
-					return "maven:3.8.4-jdk-8";
-				else
-					return "maven:latest";
-			} catch (NumberFormatException e) {
-				return "maven:3.8.4-jdk-8";
-			}		
-		} else {
-			return null;
-		}
-	}
 }

server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenModule.java

@@ -1,11 +1,7 @@
 package io.onedev.server.plugin.buildspec.maven;
 
-import com.google.common.collect.Lists;
-
 import io.onedev.commons.loader.AbstractPluginModule;
 import io.onedev.server.buildspec.job.JobSuggestion;
-import io.onedev.server.model.support.administration.GroovyScript;
-import io.onedev.server.util.ScriptContribution;
 
 /**
  * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@@ -19,19 +15,6 @@ public class MavenModule extends AbstractPluginModule {
 		
 		// put your guice bindings here
 		contribute(JobSuggestion.class, MavenJobSuggestion.class);
-		
-		contribute(ScriptContribution.class, new ScriptContribution() {
-
-			@Override
-			public GroovyScript getScript() {
-				GroovyScript script = new GroovyScript();
-				script.setName(MavenJobSuggestion.DETERMINE_DOCKER_IMAGE);
-				script.setContent(Lists.newArrayList("io.onedev.server.plugin.buildspec.maven.MavenJobSuggestion.determineDockerImage()"));
-				return script;
-			}
-			
-		});
-		
 	}
 
 }

server-plugin/server-plugin-buildspec-node/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<properties>
 		<moduleClass>io.onedev.server.plugin.buildspec.node.NodePluginModule</moduleClass>

server-plugin/server-plugin-buildspec-node/src/main/java/io/onedev/server/plugin/buildspec/node/NodeJobSuggestion.java

@@ -1,36 +1,30 @@
 package io.onedev.server.plugin.buildspec.node;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-import org.eclipse.jgit.lib.FileMode;
-import org.eclipse.jgit.lib.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
-
 import io.onedev.server.OneDev;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.buildspec.job.Job;
 import io.onedev.server.buildspec.job.JobSuggestion;
 import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
-import io.onedev.server.buildspec.step.CheckoutStep;
-import io.onedev.server.buildspec.step.CommandStep;
-import io.onedev.server.buildspec.step.SetBuildVersionStep;
+import io.onedev.server.buildspec.step.*;
 import io.onedev.server.git.Blob;
 import io.onedev.server.git.BlobIdent;
 import io.onedev.server.model.Build;
 import io.onedev.server.model.Project;
 import io.onedev.server.model.support.administration.GroovyScript;
 import io.onedev.server.util.interpolative.VariableInterpolator;
+import org.eclipse.jgit.lib.FileMode;
+import org.eclipse.jgit.lib.ObjectId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
 
 public class NodeJobSuggestion implements JobSuggestion {
 
@@ -61,9 +55,11 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.setName("angular ci");
 
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
 			
+			job.getSteps().addAll(newCacheSteps());
+			
 			SetBuildVersionStep setBuildVersion = new SetBuildVersionStep();
 			setBuildVersion.setName("set build version");
 			setBuildVersion.setBuildVersion("@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_PROJECT_VERSION + "@");
@@ -116,7 +112,6 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.getSteps().add(runCommands);
 			
 			setupTriggers(job);
-			setupCaches(job);
 			jobs.add(job);
 		} 
 		
@@ -125,9 +120,11 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.setName("react ci");
 			
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
 			
+			job.getSteps().addAll(newCacheSteps());
+			
 			SetBuildVersionStep setBuildVersion = new SetBuildVersionStep();
 			setBuildVersion.setName("set build version");
 			setBuildVersion.setBuildVersion("@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_PROJECT_VERSION + "@");
@@ -135,10 +132,9 @@ public class NodeJobSuggestion implements JobSuggestion {
 			
 			CommandStep runCommands = new CommandStep();
 			runCommands.setName("build & test");
-			runCommands.setImage("node:10.16-alpine");
+			runCommands.setImage("node");
 
 			List<String> commands = Lists.newArrayList( 
-					"npm install typescript", 
 					"npm install", 
 					"export CI=TRUE");
 
@@ -177,7 +173,6 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.getSteps().add(runCommands);
 			
 			setupTriggers(job);
-			setupCaches(job);
 			jobs.add(job);
 		} 
 		
@@ -186,9 +181,11 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.setName("vue ci");
 			
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
 			
+			job.getSteps().addAll(newCacheSteps());
+			
 			SetBuildVersionStep setBuildVersion = new SetBuildVersionStep();
 			setBuildVersion.setName("set build version");
 			setBuildVersion.setBuildVersion("@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_PROJECT_VERSION + "@");
@@ -196,7 +193,7 @@ public class NodeJobSuggestion implements JobSuggestion {
 			
 			CommandStep runCommands = new CommandStep();
 			runCommands.setName("build & test");
-			runCommands.setImage("node:10.16-alpine");
+			runCommands.setImage("node");
 
 			List<String> commands = Lists.newArrayList("npm install");
 
@@ -231,7 +228,6 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.getSteps().add(runCommands);
 			
 			setupTriggers(job);
-			setupCaches(job);
 			jobs.add(job);
 		} 
 		
@@ -240,9 +236,11 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.setName("express ci");
 			
 			CheckoutStep checkout = new CheckoutStep();
-			checkout.setName("checkout");
+			checkout.setName("checkout code");
 			job.getSteps().add(checkout);
 			
+			job.getSteps().addAll(newCacheSteps());
+			
 			SetBuildVersionStep setBuildVersion = new SetBuildVersionStep();
 			setBuildVersion.setName("set build version");
 			setBuildVersion.setBuildVersion("@" + VariableInterpolator.PREFIX_SCRIPT + GroovyScript.BUILTIN_PREFIX + DETERMINE_PROJECT_VERSION + "@");
@@ -250,7 +248,7 @@ public class NodeJobSuggestion implements JobSuggestion {
 			
 			CommandStep runCommands = new CommandStep();
 			runCommands.setName("build & test");
-			runCommands.setImage("node:10.16-alpine");
+			runCommands.setImage("node");
 
 			List<String> commands = Lists.newArrayList("npm install");
 
@@ -284,17 +282,24 @@ public class NodeJobSuggestion implements JobSuggestion {
 			job.getSteps().add(runCommands);
 			
 			setupTriggers(job);
-			setupCaches(job);
 			jobs.add(job);
 		} 
 		return jobs;
 	}
 	
-	private void setupCaches(Job job) {
-		CacheSpec cache = new CacheSpec();
-		cache.setKey("npm-cache");
-		cache.setPath("/root/.npm");
-		job.getCaches().add(cache);
+	private List<Step> newCacheSteps() {
+		var generateChecksum = new GenerateChecksumStep();
+		generateChecksum.setName("generate package checksum");
+		generateChecksum.setFiles("package-lock.json");
+		generateChecksum.setTargetFile("checksum");
+		
+		var setupCache = new SetupCacheStep();
+		setupCache.setName("set up package cache");
+		setupCache.setKey("node_modules_@file:checksum@");
+		setupCache.setPath("node_modules");
+		setupCache.getLoadKeys().add("node_modules");
+		
+		return Lists.newArrayList(generateChecksum, setupCache);
 	}
 
 	private void setupTriggers(Job job) {

server-plugin/server-plugin-executor-kubernetes/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<properties>
 		<moduleClass>io.onedev.server.plugin.executor.kubernetes.KubernetesModule</moduleClass>

server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java

@@ -55,6 +55,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static com.google.common.collect.Lists.newArrayList;
 import static io.onedev.k8shelper.KubernetesHelper.*;
 import static io.onedev.server.util.CollectionUtils.newHashMap;
 import static io.onedev.server.util.CollectionUtils.newLinkedHashMap;
@@ -901,21 +902,18 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 				
 				String containerBuildHome;
 				String containerCommandDir;
-				String containerCacheHome;
 				String containerAuthInfoDir;
 				String containerTrustCertsDir;
 				String containerWorkspace;
 				if (osInfo.isWindows()) {
 					containerBuildHome = "C:\\onedev-build";
 					containerWorkspace = containerBuildHome + "\\workspace";
-					containerCacheHome = containerBuildHome + "\\cache";
 					containerCommandDir = containerBuildHome + "\\command";
 					containerAuthInfoDir = "C:\\Users\\ContainerAdministrator\\auth-info";
 					containerTrustCertsDir = containerBuildHome + "\\trust-certs";
 				} else {
 					containerBuildHome = "/onedev-build";
 					containerWorkspace = containerBuildHome +"/workspace";
-					containerCacheHome = containerBuildHome + "/cache";
 					containerCommandDir = containerBuildHome + "/command";
 					containerAuthInfoDir = "/root/auth-info";
 					containerTrustCertsDir = containerBuildHome + "/trust-certs";
@@ -925,22 +923,20 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 						"name", "build-home", 
 						"mountPath", containerBuildHome);
 				Map<String, String> authInfoMount = newLinkedHashMap(
-						"name", "auth-info", 
-						"mountPath", containerAuthInfoDir);
+						"name", "build-home", 
+						"mountPath", containerAuthInfoDir, 
+						"subPath", "auth-info");
 				
 				// Windows nanoserver default user is ContainerUser
 				Map<String, String> authInfoMount2 = newLinkedHashMap(
-						"name", "auth-info", 
-						"mountPath", "C:\\Users\\ContainerUser\\auth-info");
-				
-				Map<String, String> cacheHomeMount = newLinkedHashMap(
-						"name", "cache-home", 
-						"mountPath", containerCacheHome);
+						"name", "build-home", 
+						"mountPath", "C:\\Users\\ContainerUser\\auth-info", 
+						"subPath", "auth-info");
 				Map<String, String> trustCertsMount = newLinkedHashMap(
 						"name", "trust-certs", 
 						"mountPath", containerTrustCertsDir);
 				
-				var commonVolumeMounts = Lists.newArrayList(buildHomeMount, authInfoMount, cacheHomeMount);
+				var commonVolumeMounts = newArrayList(buildHomeMount, authInfoMount);
 				if (osInfo.isWindows())
 					commonVolumeMounts.add(authInfoMount2);
 				if (trustCertsConfigMapName != null)
@@ -952,12 +948,12 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 				} else {
 					List<Action> actions = new ArrayList<>();
 					CommandFacade facade = new CommandFacade((String) executionContext, null,
-							Lists.newArrayList("this does not matter"), false);
+							newArrayList("this does not matter"), false);
 					actions.add(new Action("test", facade, ExecuteCondition.ALWAYS));
 					entryFacade = new CompositeFacade(actions);
 				}
 				
-				List<String> containerNames = Lists.newArrayList("init");
+				List<String> containerNames = newArrayList("init");
 				
 				String helperImageSuffix;
 				if (osInfo.isWindows()) {  
@@ -988,6 +984,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 						"value", containerWorkspace
 						));
 
+				Collection<String> cachePaths = new HashSet<>();
 				entryFacade.traverse((facade, position) -> {
 					String containerName = getContainerName(position);
 					containerNames.add(containerName);
@@ -1005,48 +1002,58 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 								"image", mapImage(execution.getImage()));
 						if (commandFacade.isUseTTY())
 							stepContainerSpec.put("tty", true);
-						stepContainerSpec.put("volumeMounts", commonVolumeMounts);
+						var volumeMounts = buildVolumeMounts(cachePaths);
+						volumeMounts.addAll(commonVolumeMounts);
+						stepContainerSpec.put("volumeMounts", volumeMounts);
 						stepContainerSpec.put("env", commonEnvs);
 					} else if (facade instanceof BuildImageFacade) {
 						throw new ExplicitException("This step can only be executed by server docker executor or " +
 								"remote docker executor. Use kaniko step instead to build image in kubernetes cluster");
-					} else if (facade instanceof RunContainerFacade) {
+					} else if (facade instanceof RunContainerFacade || facade instanceof RunImagetoolsFacade) {
 						throw new ExplicitException("This step can only be executed by server docker executor or " +
 								"remote docker executor");
-					} else { 
+					} else {
+						if (facade instanceof SetupCacheFacade) {
+							var cachePath = ((SetupCacheFacade) facade).getPath();
+							if (!cachePaths.add(cachePath))
+								throw new ExplicitException("Duplicate cache path: " + cachePath);
+						}
 						stepContainerSpec = newHashMap(
 								"name", containerName, 
 								"image", helperImage);
-						stepContainerSpec.put("volumeMounts", commonVolumeMounts);
+						var volumeMounts = buildVolumeMounts(cachePaths);
+						volumeMounts.addAll(commonVolumeMounts);
+						stepContainerSpec.put("volumeMounts", volumeMounts);
 						stepContainerSpec.put("env", commonEnvs);
 					}
 					
-					String positionStr = stringifyStepPosition(position);
-					if (osInfo.isLinux()) {
-						stepContainerSpec.put("command", Lists.newArrayList("sh"));
-						stepContainerSpec.put("args", Lists.newArrayList(containerCommandDir + "/" + positionStr + ".sh"));
-					} else {
-						stepContainerSpec.put("command", Lists.newArrayList("cmd"));
-						stepContainerSpec.put("args", Lists.newArrayList("/c", containerCommandDir + "\\" + positionStr + ".bat"));
-					}
+					if (stepContainerSpec != null) {
+						String positionStr = stringifyStepPosition(position);
+						if (osInfo.isLinux()) {
+							stepContainerSpec.put("command", newArrayList("sh"));
+							stepContainerSpec.put("args", newArrayList(containerCommandDir + "/" + positionStr + ".sh"));
+						} else {
+							stepContainerSpec.put("command", newArrayList("cmd"));
+							stepContainerSpec.put("args", newArrayList("/c", containerCommandDir + "\\" + positionStr + ".bat"));
+						}
 
-					Map<Object, Object> requestsSpec = newLinkedHashMap(
-									"cpu", "0", 
-									"memory", "0");
-					Map<Object, Object> limitsSpec = new LinkedHashMap<>();
-					if (getCpuLimit() != null)
-						limitsSpec.put("cpu", getCpuLimit());
-					if (getMemoryLimit() != null)
-						limitsSpec.put("memory", getMemoryLimit());
-					if (!limitsSpec.isEmpty()) {
-						stepContainerSpec.put(
-								"resources", newLinkedHashMap(
-										"limits", limitsSpec, 
-										"requests", requestsSpec));
+						Map<Object, Object> requestsSpec = newLinkedHashMap(
+								"cpu", "0",
+								"memory", "0");
+						Map<Object, Object> limitsSpec = new LinkedHashMap<>();
+						if (getCpuLimit() != null)
+							limitsSpec.put("cpu", getCpuLimit());
+						if (getMemoryLimit() != null)
+							limitsSpec.put("memory", getMemoryLimit());
+						if (!limitsSpec.isEmpty()) {
+							stepContainerSpec.put(
+									"resources", newLinkedHashMap(
+											"limits", limitsSpec,
+											"requests", requestsSpec));
+						}
+
+						containerSpecs.add(stepContainerSpec);
 					}
-					
-					containerSpecs.add(stepContainerSpec);
-					
 					return null;
 				}, new ArrayList<>());
 				
@@ -1057,32 +1064,35 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 					k8sHelperClassPath = "C:\\k8s-helper\\*";
 				}
 				
-				List<String> sidecarArgs = Lists.newArrayList(
+				List<String> sidecarArgs = newArrayList(
 						"-classpath", k8sHelperClassPath,
 						"io.onedev.k8shelper.SideCar");
-				List<String> initArgs = Lists.newArrayList(
+				List<String> initArgs = newArrayList(
 						"-classpath", k8sHelperClassPath, 
 						"io.onedev.k8shelper.Init");
 				if (jobContext == null) {
 					sidecarArgs.add("test");
 					initArgs.add("test");
 				}
+
+				var volumeMounts = buildVolumeMounts(cachePaths);
+				volumeMounts.addAll(commonVolumeMounts);
 				
 				Map<Object, Object> initContainerSpec = newHashMap(
 						"name", "init", 
 						"image", helperImage, 
-						"command", Lists.newArrayList("java"), 
+						"command", newArrayList("java"), 
 						"args", initArgs,
 						"env", commonEnvs,
-						"volumeMounts", commonVolumeMounts);
+						"volumeMounts", volumeMounts);
 				
 				Map<Object, Object> sidecarContainerSpec = newLinkedHashMap(
 						"name", "sidecar", 
 						"image", helperImage, 
-						"command", Lists.newArrayList("java"), 
+						"command", newArrayList("java"), 
 						"args", sidecarArgs, 
 						"env", commonEnvs, 
-						"volumeMounts", commonVolumeMounts);
+						"volumeMounts", volumeMounts);
 				
 				sidecarContainerSpec.put("resources", newLinkedHashMap("requests", newLinkedHashMap(
 						"cpu", getCpuRequest(), 
@@ -1104,22 +1114,13 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 				Map<Object, Object> buildHomeVolume = newLinkedHashMap(
 						"name", "build-home", 
 						"emptyDir", newLinkedHashMap());
-				Map<Object, Object> userHomeVolume = newLinkedHashMap(
-						"name", "auth-info", 
-						"emptyDir", newLinkedHashMap());
-				Map<Object, Object> cacheHomeVolume = newLinkedHashMap(
-						"name", "cache-home", 
-						"hostPath", newLinkedHashMap(
-								"path", osInfo.getCacheHome(), 
-								"type", "DirectoryOrCreate"));
-				List<Object> volumes = Lists.<Object>newArrayList(buildHomeVolume, userHomeVolume, cacheHomeVolume);
+				List<Object> volumes = newArrayList(buildHomeVolume);
 				if (trustCertsConfigMapName != null) {
 					volumes.add(newLinkedHashMap(
 							"name", "trust-certs", 
 							"configMap", newLinkedHashMap(
 									"name", trustCertsConfigMapName)));
 				}
-				
 				podSpec.put("volumes", volumes);
 
 				Map<Object, Object> podDef = newLinkedHashMap(
@@ -1260,6 +1261,20 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
 		}
 	}
 	
+	private List<Object> buildVolumeMounts(Collection<String> cachePaths) {
+		var volumeMounts = new ArrayList<>();
+		int index = 1;
+		for (var cachePath: cachePaths) {
+			var volumeMount = newLinkedHashMap(
+					"name", "build-home",
+					"mountPath", cachePath,
+					"subPath", "cache/" + index);
+			volumeMounts.add(volumeMount);
+			index++;
+		}
+		return volumeMounts;
+	}
+	
 	private String getContainerName(List<Integer> stepPosition) {
 		return "step-" + stringifyStepPosition(stepPosition);
 	}

server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java

@@ -1,18 +1,21 @@
 package io.onedev.server.plugin.executor.kubernetes;
 
-import io.onedev.commons.utils.ExplicitException;
+import com.google.common.base.Splitter;
 import io.onedev.commons.utils.FileUtils;
 import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TarUtils;
 import io.onedev.commons.utils.TaskLogger;
-import io.onedev.k8shelper.CacheAllocationRequest;
 import io.onedev.k8shelper.K8sJobData;
 import io.onedev.k8shelper.KubernetesHelper;
+import io.onedev.server.entitymanager.JobCacheManager;
+import io.onedev.server.entitymanager.ProjectManager;
 import io.onedev.server.job.JobContext;
 import io.onedev.server.job.JobManager;
 import io.onedev.server.persistence.SessionManager;
 import io.onedev.server.rest.annotation.Api;
 import io.onedev.server.security.SecurityUtils;
 import org.apache.commons.lang.SerializationUtils;
+import org.apache.shiro.authz.UnauthorizedException;
 
 import javax.annotation.Nullable;
 import javax.inject.Inject;
@@ -43,22 +46,30 @@ public class KubernetesResource {
 
 	private final JobManager jobManager;
 	
+	private final JobCacheManager jobCacheManager;
+	
 	private final SessionManager sessionManager;
 	
+	private final ProjectManager projectManager;
+	
     @Context
     private HttpServletRequest request;
     
     @Inject
-    public KubernetesResource(JobManager jobManager, SessionManager sessionManager) {
+    public KubernetesResource(JobManager jobManager, JobCacheManager jobCacheManager, 
+							  SessionManager sessionManager, ProjectManager projectManager) {
     	this.jobManager = jobManager;
+		this.jobCacheManager = jobCacheManager;
     	this.sessionManager = sessionManager;
+		this.projectManager = projectManager;
 	}
     
 	@Path("/job-data")
 	@Produces(MediaType.APPLICATION_OCTET_STREAM)
-    @POST
-    public byte[] getJobData(@Nullable String jobWorkspace) {
-		JobContext jobContext = jobManager.getJobContext(getJobToken(), true);
+    @GET
+    public byte[] getJobData(@QueryParam("jobToken") String jobToken, 
+							 @QueryParam("jobWorkspace") @Nullable String jobWorkspace) {
+		JobContext jobContext = jobManager.getJobContext(jobToken, true);
 		if (StringUtils.isNotBlank(jobWorkspace))
 			jobManager.reportJobWorkspace(jobContext, jobWorkspace);	
 		K8sJobData k8sJobData = new K8sJobData(
@@ -69,23 +80,14 @@ public class KubernetesResource {
 		return SerializationUtils.serialize(k8sJobData);
     }
 	
-	@Path("/allocate-caches")
-	@Consumes(MediaType.APPLICATION_OCTET_STREAM)
-	@Produces(MediaType.APPLICATION_OCTET_STREAM)
-    @POST
-    public byte[] allocateCaches(String requestString) {
-		CacheAllocationRequest request = CacheAllocationRequest.fromString(requestString);
-		return SerializationUtils.serialize((Serializable) jobManager.allocateCaches(
-				jobManager.getJobContext(getJobToken(), true), request));
-    }
-	
 	@Path("/run-server-step")
 	@Consumes(MediaType.APPLICATION_OCTET_STREAM)
 	@POST
-	public Response runServerStep(InputStream is) {
+	public Response runServerStep(@QueryParam("jobToken") String jobToken, InputStream is) {
+		JobContext jobContext = jobManager.getJobContext(jobToken, true);
 		// Make sure we are not occupying a database connection here as we will occupy 
 		// database connection when running step at project server side
-		sessionManager.closeSession(); 
+		sessionManager.closeSession();
 		try {
 			StreamingOutput os = output -> {
 				File filesDir = FileUtils.createTempDir();
@@ -100,9 +102,8 @@ public class KubernetesResource {
 					for (int i=0; i<length; i++) 
 						placeholderValues.put(readString(is), readString(is));
 					
-					FileUtils.untar(is, filesDir, false);
+					TarUtils.untar(is, filesDir, false);
 					
-					JobContext jobContext = jobManager.getJobContext(getJobToken(), true);
 					Map<String, byte[]> outputFiles = jobManager.runServerStep(jobContext, 
 							stepPosition, filesDir, placeholderValues, true, new TaskLogger() {
 
@@ -139,21 +140,86 @@ public class KubernetesResource {
 	@Path("/download-dependencies")
 	@Produces(MediaType.APPLICATION_OCTET_STREAM)
 	@GET
-	public Response downloadDependencies() {
-		StreamingOutput os = output -> {
-			JobContext jobContext = jobManager.getJobContext(getJobToken(), true);
-			File tempDir = FileUtils.createTempDir();
-			try {
-				jobManager.copyDependencies(jobContext, tempDir);
-				FileUtils.tar(tempDir, output, false);
-				output.flush();
-			} finally {
-				FileUtils.deleteDir(tempDir);
-			}
-	   };
-		return Response.ok(os).build();
+	public Response downloadDependencies(@QueryParam("jobToken") String jobToken) {
+		sessionManager.closeSession();
+		try {
+			StreamingOutput output = os -> {
+				JobContext jobContext = jobManager.getJobContext(jobToken, true);
+				File tempDir = FileUtils.createTempDir();
+				try {
+					jobManager.copyDependencies(jobContext, tempDir);
+					TarUtils.tar(tempDir, os, false);
+					os.flush();
+				} finally {
+					FileUtils.deleteDir(tempDir);
+				}
+			};
+			return Response.ok(output).build();
+		} finally {
+			sessionManager.openSession();
+		}
 	}
-	 
+
+	@Path("/download-cache")
+	@Produces(MediaType.APPLICATION_OCTET_STREAM)
+	@GET
+	public Response downloadCache(
+			@QueryParam("jobToken") String jobToken,
+			@QueryParam("cacheKey") @Nullable String cacheKey, 
+			@QueryParam("cacheLoadKeys") @Nullable String joinedCacheLoadKeys, 
+			@QueryParam("cachePath") String cachePath) {
+		sessionManager.closeSession();
+		try {
+			StreamingOutput output = os -> {
+				var jobContext = jobManager.getJobContext(jobToken, true);
+				if (cacheKey != null) {
+					jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, cachePath, os);
+				} else {
+					var cacheLoadKeys = Splitter.on('\n').splitToList(joinedCacheLoadKeys);
+					jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, cachePath, os);
+				}
+			};
+			return Response.ok(output).build();
+		} finally {
+			sessionManager.openSession();
+		}
+	}
+	
+	@Path("/upload-cache")
+	@GET
+	public Response checkUploadCache(
+			@QueryParam("jobToken") String jobToken,
+			@QueryParam("cacheKey") String cacheKey,
+			@QueryParam("cachePath") String cachePath) {
+		var jobContext = jobManager.getJobContext(jobToken, true);
+		var project = projectManager.load(jobContext.getProjectId());
+		if (project.isCommitOnBranch(jobContext.getCommitId(), project.getDefaultBranch())
+				|| SecurityUtils.canUploadCache(project)) {
+			return Response.ok().build();
+		} else {
+			throw new UnauthorizedException("Not authorized to upload cache");
+		}
+	}
+
+	@Path("/upload-cache")
+	@Consumes(MediaType.APPLICATION_OCTET_STREAM)
+	@POST
+	public Response uploadCache(
+			@QueryParam("jobToken") String jobToken,
+			@QueryParam("cacheKey") String cacheKey, 
+			@QueryParam("cachePath") String cachePath, 
+			InputStream is) {
+		checkUploadCache(jobToken, cacheKey, cachePath);
+		var jobContext = jobManager.getJobContext(jobToken, true);
+		sessionManager.closeSession();
+		try {
+			jobCacheManager.uploadCache(jobContext.getProjectId(), cacheKey, cachePath, is);
+			return Response.ok().build();
+		} finally {
+			sessionManager.openSession();
+		}
+	}	
+	
 	@GET
 	@Path("/test")
 	public Response test() {
@@ -164,12 +230,4 @@ public class KubernetesResource {
 			return Response.status(400).entity("Missing job token").build();
 	}
 	
-	private String getJobToken() {
-		String jobToken = SecurityUtils.getBearerToken(request);
-		if (jobToken != null)
-			return jobToken;
-		else
-			throw new ExplicitException("Job token is expected");
-	}
-	
 }

server-plugin/server-plugin-executor-remotedocker/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-executor-remoteshell/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<dependencies>
 		<dependency>

server-plugin/server-plugin-executor-remoteshell/src/main/java/io/onedev/server/plugin/executor/remoteshell/RemoteShellExecutor.java

@@ -10,7 +10,6 @@ import io.onedev.server.OneDev;
 import io.onedev.server.annotation.Editable;
 import io.onedev.server.annotation.Horizontal;
 import io.onedev.server.annotation.Numeric;
-import io.onedev.server.buildspec.job.CacheSpec;
 import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.entitymanager.AgentManager;
 import io.onedev.server.job.*;
@@ -24,7 +23,6 @@ import io.onedev.server.terminal.Shell;
 import io.onedev.server.terminal.Terminal;
 import org.eclipse.jetty.websocket.api.Session;
 
-import java.io.File;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
 
@@ -100,13 +98,6 @@ public class RemoteShellExecutor extends ServerShellExecutor {
 								+ "by docker aware executors");
 					}
 
-					for (CacheSpec cacheSpec : jobContext.getCacheSpecs()) {
-						if (new File(cacheSpec.getPath()).isAbsolute()) {
-							throw new ExplicitException("Shell executor does not support "
-									+ "absolute cache path: " + cacheSpec.getPath());
-						}
-					}
-
 					String jobToken = jobContext.getJobToken();
 					ShellJobData jobData = new ShellJobData(jobToken, getName(), jobContext.getProjectPath(),
 							jobContext.getProjectId(), jobContext.getRefName(), jobContext.getCommitId().name(),

server-plugin/server-plugin-executor-serverdocker/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>io.onedev</groupId>
 		<artifactId>server-plugin</artifactId>
-		<version>10.0.0</version>
+		<version>10.1.0</version>
 	</parent>
 	<properties>
 		<moduleClass>io.onedev.server.plugin.executor.serverdocker.ServerDockerModule</moduleClass>

server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java

@@ -21,10 +21,7 @@ import io.onedev.server.cluster.ClusterManager;
 import io.onedev.server.cluster.ClusterRunnable;
 import io.onedev.server.entitymanager.SettingManager;
 import io.onedev.server.git.location.GitLocation;
-import io.onedev.server.job.JobContext;
-import io.onedev.server.job.JobManager;
-import io.onedev.server.job.JobRunnable;
-import io.onedev.server.job.ResourceAllocator;
+import io.onedev.server.job.*;
 import io.onedev.server.model.support.ImageMapping;
 import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;
 import io.onedev.server.model.support.administration.jobexecutor.RegistryLogin;
@@ -59,8 +56,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 	private static final long serialVersionUID = 1L;
 	
 	static final int ORDER = 50;
-
-	private static final Object cacheHomeCreationLock = new Object();
 	
 	private List<RegistryLogin> registryLogins = new ArrayList<>();
 
@@ -217,19 +212,15 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 		useDockerSock(docker, getDockerSockPath());
 		return docker;
 	}
-	
-	private File getCacheHome(JobExecutor jobExecutor) {
-		File file = new File(Bootstrap.getSiteDir(), "cache/" + jobExecutor.getName());
-		if (!file.exists()) synchronized (cacheHomeCreationLock) {
-			FileUtils.createDir(file);
-		}
-		return file;
-	}
 
 	private ClusterManager getClusterManager() {
 		return OneDev.getInstance(ClusterManager.class);
 	}
 	
+	private SettingManager getSettingManager() {
+		return OneDev.getInstance(SettingManager.class);
+	}
+	
 	private JobManager getJobManager() {
 		return OneDev.getInstance(JobManager.class);
 	}
@@ -269,36 +260,18 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 						String network = getName() + "-" + jobContext.getProjectId() + "-"
 								+ jobContext.getBuildNumber() + "-" + jobContext.getRetried();
 
-						String serverAddress = getClusterManager().getLocalServerAddress();
+						String localServer = getClusterManager().getLocalServerAddress();
 						jobLogger.log(String.format("Executing job (executor: %s, server: %s, network: %s)...", 
-								getName(), serverAddress, network));
+								getName(), localServer, network));
 
-						File hostCacheHome = getCacheHome(jobContext.getJobExecutor());
-
-						jobLogger.log("Setting up job cache...");
-						JobCache cache = new JobCache(hostCacheHome) {
-
-							@Override
-							protected Map<CacheInstance, String> allocate(CacheAllocationRequest request) {
-								return getJobManager().allocateCaches(jobContext, request);
-							}
-
-							@Override
-							protected void delete(File cacheDir) {
-								deleteDir(cacheDir, newDocker(), Bootstrap.isInDocker());
-							}
-
-						};
-						cache.init(false);
-						
 						createNetwork(newDocker(), network, getNetworkOptions(), jobLogger);
 						try {
 							OsInfo osInfo = OneDev.getInstance(OsInfo.class);
 
-							var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
+							var serverUrl = getSettingManager().getSystemSetting().getServerUrl();
 							for (var jobService : jobContext.getServices()) {
 								var docker = newDocker();
-								var builtInRegistryLogin = new BuiltInRegistryLogin(builtInRegistryUrl, 
+								var builtInRegistryLogin = new BuiltInRegistryLogin(serverUrl, 
 										jobContext.getJobToken(), jobService.getBuiltInRegistryAccessToken());
 								callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
 									startService(docker, network, jobService, osInfo, getImageMappingFacades(),
@@ -310,10 +283,10 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 							File hostWorkspace = new File(hostBuildHome, "workspace");
 							FileUtils.createDir(hostWorkspace);
 
+							var cacheHelper = new ServerCacheHelper(hostBuildHome, jobContext, jobLogger);
+							
 							AtomicReference<File> hostAuthInfoDir = new AtomicReference<>(null);
 							try {
-								cache.installSymbolinks(hostWorkspace);
-
 								jobLogger.log("Copying job dependencies...");
 								getJobManager().copyDependencies(jobContext, hostWorkspace);
 
@@ -339,8 +312,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 																 @Nullable String workingDir, Map<String, String> volumeMounts,
 																 List<Integer> position, boolean useTTY) {
 										image = mapImage(image);
-										// Uninstall symbol links as docker can not process it well
-										cache.uninstallSymbolinks(hostWorkspace);
 										containerName = network + "-step-" + stringifyStepPosition(position);
 										try {
 											var useProcessIsolation = isUseProcessIsolation(docker, image, osInfo, jobLogger);
@@ -368,11 +339,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 											else if (workingDir != null) 
 												docker.addArgs("-w", workingDir);
 
-											for (Map.Entry<CacheInstance, String> entry : cache.getAllocations().entrySet()) {
-												String hostCachePath = new File(hostCacheHome, entry.getKey().toString()).getAbsolutePath();
-												String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());
-												docker.addArgs("-v", getHostPath(hostCachePath) + ":" + containerCachePath);
-											}
+											cacheHelper.mountVolumes(docker, ServerDockerExecutor.this::getHostPath);
 
 											if (isMountDockerSock()) {
 												if (getDockerSockPath() != null) {
@@ -422,7 +389,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 											return result.getReturnCode();													
 										} finally {
 											containerName = null;
-											cache.installSymbolinks(hostWorkspace);
 										}
 									}
 
@@ -447,8 +413,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 														hostBuildHome, commandFacade, osInfo, hostAuthInfoDir.get() != null);
 												
 												var docker = newDocker();
-												var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
-												var builtInRegistryLogin = new BuiltInRegistryLogin(builtInRegistryUrl,
+												var builtInRegistryLogin = new BuiltInRegistryLogin(serverUrl,
 														jobContext.getJobToken(), commandFacade.getBuiltInRegistryAccessToken());
 												int exitCode = callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
 													return runStepContainer(docker, execution.getImage(), entrypoint.executable(),
@@ -464,8 +429,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 											} else if (facade instanceof BuildImageFacade) {
 												var buildImageFacade = (BuildImageFacade) facade;
 												var docker = newDocker();
-												var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
-												var builtInRegistryLogin = new BuiltInRegistryLogin(builtInRegistryUrl, 
+												var builtInRegistryLogin = new BuiltInRegistryLogin(serverUrl,
 														jobContext.getJobToken(), buildImageFacade.getBuiltInRegistryAccessToken());
 												callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
 													buildImage(docker, buildImageFacade, hostBuildHome, jobLogger);
@@ -474,8 +438,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 											} else if (facade instanceof RunImagetoolsFacade) {
 												var runImagetoolsFacade = (RunImagetoolsFacade) facade;
 												var docker = newDocker();
-												var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
-												var builtInRegistryLogin = new BuiltInRegistryLogin(builtInRegistryUrl,
+												var builtInRegistryLogin = new BuiltInRegistryLogin(serverUrl,
 														jobContext.getJobToken(), runImagetoolsFacade.getBuiltInRegistryAccessToken());
 												callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
 													runImagetools(docker, runImagetoolsFacade, hostBuildHome, jobLogger);
@@ -493,8 +456,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 												if (container.getArgs() != null)
 													arguments.addAll(Arrays.asList(StringUtils.parseQuoteTokens(container.getArgs())));
 												var docker = newDocker();
-												var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
-												var builtInRegistryLogin =new BuiltInRegistryLogin(builtInRegistryUrl, 
+												var builtInRegistryLogin =new BuiltInRegistryLogin(serverUrl, 
 														jobContext.getJobToken(), runContainerFacade.getBuiltInRegistryAccessToken());
 												int exitCode = callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
 													return runStepContainer(docker, container.getImage(), null, options, arguments,
@@ -510,9 +472,9 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 												try {
 													CheckoutFacade checkoutFacade = (CheckoutFacade) facade;
 													jobLogger.log("Checking out code...");
-													
+
 													Commandline git = new Commandline(AppLoader.getInstance(GitLocation.class).getExecutable());
-													
+
 													if (hostAuthInfoDir.get() == null)
 														hostAuthInfoDir.set(FileUtils.createTempDir());
 													git.environments().put("HOME", hostAuthInfoDir.get().getAbsolutePath());
@@ -527,19 +489,19 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 													File trustCertsFile = new File(hostBuildHome, "trust-certs.pem");
 													installGitCert(git, Bootstrap.getTrustCertsDir(),
 															trustCertsFile, containerTrustCerts,
-															ExecutorUtils.newInfoLogger(jobLogger), 
+															ExecutorUtils.newInfoLogger(jobLogger),
 															ExecutorUtils.newWarningLogger(jobLogger));
-													
+
 													CloneInfo cloneInfo = checkoutFacade.getCloneInfo();
-													cloneInfo.writeAuthData(hostAuthInfoDir.get(), git, true, 
-															ExecutorUtils.newInfoLogger(jobLogger), 
+													cloneInfo.writeAuthData(hostAuthInfoDir.get(), git, true,
+															ExecutorUtils.newInfoLogger(jobLogger),
 															ExecutorUtils.newWarningLogger(jobLogger));
 
 													if (trustCertsFile.exists())
 														git.addArgs("-c", "http.sslCAInfo=" + trustCertsFile.getAbsolutePath());
 
 													int cloneDepth = checkoutFacade.getCloneDepth();
-			
+
 													cloneRepository(git, jobContext.getProjectGitDir(), cloneInfo.getCloneUrl(),
 															jobContext.getRefName(), jobContext.getCommitId().name(),
 															checkoutFacade.isWithLfs(), checkoutFacade.isWithSubmodules(),
@@ -549,7 +511,10 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 													jobLogger.error("Step \"" + stepNames + "\" is failed (" + DateUtils.formatDuration(duration) + "): " + getErrorMessage(e));
 													return false;
 												}
-											} else {
+											} else if (facade instanceof SetupCacheFacade) {
+												SetupCacheFacade setupCacheFacade = (SetupCacheFacade) facade;
+												cacheHelper.setupCache(setupCacheFacade);
+											} else if (facade instanceof ServerSideFacade) {
 												ServerSideFacade serverSideFacade = (ServerSideFacade) facade;
 												try {
 													serverSideFacade.execute(hostBuildHome, new ServerSideFacade.Runner() {
@@ -563,11 +528,13 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 													});
 												} catch (Exception e) {
 													if (ExceptionUtils.find(e, InterruptedException.class) == null) {
-														long duration = System.currentTimeMillis() - time;														
+														long duration = System.currentTimeMillis() - time;
 														jobLogger.error("Step \"" + stepNames + "\" is failed: (" + DateUtils.formatDuration(duration) + ") " + getErrorMessage(e));
 													}
 													return false;
 												}
+											} else {
+												throw new ExplicitException("Unexpected step type: " + facade.getClass());
 											}
 											long duration = System.currentTimeMillis() - time;
 											jobLogger.success("Step \"" + stepNames + "\" is successful (" + DateUtils.formatDuration(duration) + ")");
@@ -584,13 +551,15 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 
 								}, new ArrayList<>());
 
-								if (!successful)
+								if (successful) 
+									cacheHelper.uploadCaches();
+								else 
 									throw new FailedException();
 							} finally {
-								cache.uninstallSymbolinks(hostWorkspace);
 								// Fix https://code.onedev.io/onedev/server/~issues/597
-								if (SystemUtils.IS_OS_WINDOWS)
+								if (SystemUtils.IS_OS_WINDOWS) {
 									FileUtils.deleteDir(hostWorkspace);
+								}
 								if (hostAuthInfoDir.get() != null)
 									FileUtils.deleteDir(hostAuthInfoDir.get());
 							}
@@ -702,15 +671,10 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 	@Override
 	public void test(TestData testData, TaskLogger jobLogger) {
 		var docker = newDocker();
-		var builtInRegistryUrl = OneDev.getInstance(SettingManager.class).getSystemSetting().getServerUrl();
 		callWithDockerAuth(docker, getRegistryLoginFacades(), null, () -> {
 			File workspaceDir = null;
-			File cacheDir = null;
 			try {
 				workspaceDir = FileUtils.createTempDir("workspace");
-				cacheDir = new File(getCacheHome(ServerDockerExecutor.this), UUID.randomUUID().toString());
-				FileUtils.createDir(cacheDir);
-
 				jobLogger.log("Testing specified docker image...");
 				docker.clearArgs();
 				docker.addArgs("run", "--rm");
@@ -721,16 +685,11 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 				if (getRunOptions() != null)
 					docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
 				String containerWorkspacePath;
-				String containerCachePath;
-				if (SystemUtils.IS_OS_WINDOWS) {
+				if (SystemUtils.IS_OS_WINDOWS) 
 					containerWorkspacePath = "C:\\onedev-build\\workspace";
-					containerCachePath = "C:\\onedev-build\\cache";
-				} else {
+				else 
 					containerWorkspacePath = "/onedev-build/workspace";
-					containerCachePath = "/onedev-build/cache";
-				}
 				docker.addArgs("-v", getHostPath(workspaceDir.getAbsolutePath()) + ":" + containerWorkspacePath);
-				docker.addArgs("-v", getHostPath(cacheDir.getAbsolutePath()) + ":" + containerCachePath);
 
 				docker.addArgs("-w", containerWorkspacePath);
 				docker.addArgs(testData.getDockerImage());
@@ -758,8 +717,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
 			} finally {
 				if (workspaceDir != null)
 					FileUtils.deleteDir(workspaceDir);
-				if (cacheDir != null)
-					FileUtils.deleteDir(cacheDir);
 			}
 
 			if (!SystemUtils.IS_OS_WINDOWS) {