archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

feat: Run rootless docker image in jobs steps (#899)

Browse Source
This commit is contained in:
Robin Shen 2024-03-03 11:42:21 +08:00
parent fc6316530c
commit 6ff40b2c3c
17 changed files with 159 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -630,8 +630,8 @@
</repository>
</repositories>
<properties>
<commons.version>2.8.7</commons.version>
<agent.version>1.10.10</agent.version>
<commons.version>2.8.8</commons.version>
<agent.version>1.10.11</agent.version>
<slf4j.version>2.0.9</slf4j.version>
<logback.version>1.4.14</logback.version>
<antlr.version>4.7.2</antlr.version>

17
server-core/src/main/java/io/onedev/server/buildspec/Service.java
View File

@ -25,6 +25,8 @@ public class Service implements NamedElement, Serializable {
private String image;
private String runAs;
private String arguments;
private List<EnvVar> envVars = new ArrayList<>();
@ -68,6 +70,18 @@ public class Service implements NamedElement, Serializable {
this.image = image;
}
@Editable(order=210, name="Run As", placeholder = "root", description = "Optionally specify uid:gid to run container as. " +
"<b class='text-warning'>Note:</b> This setting should be left empty if underlying container facility is " +
"rootless or use user namespace remapping")
@RegEx(pattern="\\d+:\\d+", message = "Should be specified in form of <uid>:<gid>")
public String getRunAs() {
return runAs;
}
public void setRunAs(String runAs) {
this.runAs = runAs;
}
@Editable(order=220, description="Optionally specify arguments to run above image")
@Interpolative(variableSuggester="suggestVariables")
public String getArguments() {
@ -125,7 +139,8 @@ public class Service implements NamedElement, Serializable {
var envs = new HashMap<String, String>();
for (var envVar: getEnvVars())
envs.put(envVar.getName(), envVar.getValue());
return new ServiceFacade(getName(), getImage(), getArguments(), envs, getReadinessCheckCommand(), getBuiltInRegistryAccessTokenSecret());
return new ServiceFacade(getName(), getImage(), getRunAs(), getArguments(), envs,
getReadinessCheckCommand(), getBuiltInRegistryAccessTokenSecret());
}
}

12
server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java
View File

@ -16,7 +16,6 @@ import io.onedev.server.model.support.administration.jobexecutor.RegistryLoginAw
import io.onedev.server.util.UrlUtils;
import javax.validation.constraints.NotEmpty;
import java.util.ArrayList;
import java.util.List;
import static io.onedev.agent.DockerExecutorUtils.buildDockerConfig;
@ -54,7 +53,12 @@ public class BuildImageWithKanikoStep extends CommandStep {
public boolean isUseTTY() {
return true;
}
@Override
public String getRunAs() {
return null;
}
@Editable(order=100, description="Optionally specify build context path relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a>. "
+ "Leave empty to use job workspace itself. The file <code>Dockerfile</code> is expected to exist in build context " +
"directory, unless you specify a different location with option <code>--dockerfile</code>")
@ -134,7 +138,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
return new DefaultInterpreter() {
@Override
public CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, String image,
public CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, String image, String runAs,
String builtInRegistryAccessToken, boolean useTTY) {
var commandsBuilder = new StringBuilder();
if (jobExecutor instanceof RegistryLoginAware) {
@ -163,7 +167,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
commandsBuilder.append(" ").append(getMoreOptions());
commandsBuilder.append("\n");
return new CommandFacade(image, builtInRegistryAccessToken, commandsBuilder.toString(), useTTY);
return new CommandFacade(image, runAs, builtInRegistryAccessToken, commandsBuilder.toString(), useTTY);
}
};

39
server-core/src/main/java/io/onedev/server/buildspec/step/CommandStep.java
View File

@ -34,15 +34,13 @@ public class CommandStep extends Step {
private String image;
private String runAsUser;
private String runAsGroup;
private String runAs;
private String builtInRegistryAccessTokenSecret;
private Interpreter interpreter = new DefaultInterpreter();
private boolean useTTY;
private boolean useTTY = true;
@Editable(order=50, description="Whether or not to run this step inside container")
public boolean isRunInContainer() {
@ -70,26 +68,17 @@ public class CommandStep extends Step {
this.image = image;
}
@Editable(order=103, description="Optionally specify uid to run the container")
@Editable(order=105, name="Run As", placeholder = "root", description = "Optionally specify uid:gid to run container as. " +
"<b class='text-warning'>Note:</b> This setting should be left empty if underlying container facility is " +
"rootless or use user namespace remapping")
@ShowCondition("isRunInContainerEnabled")
@Numeric
public String getRunAsUser() {
return runAsUser;
@RegEx(pattern="\\d+:\\d+", message = "Should be specified in form of <uid>:<gid>")
public String getRunAs() {
return runAs;
}
public void setRunAsUser(String runAsUser) {
this.runAsUser = runAsUser;
}
@Editable(order=107, description="Optionally specify gid to run the container")
@ShowCondition("isRunInContainerEnabled")
@Numeric
public String getRunAsGroup() {
return runAsGroup;
}
public void setRunAsGroup(String runAsGroup) {
this.runAsGroup = runAsGroup;
public void setRunAs(String runAs) {
this.runAs = runAs;
}
static List<InputSuggestion> suggestVariables(String matchWith) {
@ -105,7 +94,7 @@ public class CommandStep extends Step {
public void setInterpreter(Interpreter interpreter) {
this.interpreter = interpreter;
}
@Editable(order=9000, name="Built-in Registry Access Token Secret", group = "More Settings",
descriptionProvider = "getBuiltInRegistryAccessTokenSecretDescription")
@ChoiceProvider("getAccessTokenSecretChoices")
@ -148,9 +137,9 @@ public class CommandStep extends Step {
builtInRegistryAccessToken = build.getJobAuthorizationContext().getSecretValue(getBuiltInRegistryAccessTokenSecret());
else
builtInRegistryAccessToken = null;
return getInterpreter().getExecutable(jobExecutor, jobToken, getImage(), builtInRegistryAccessToken, isUseTTY());
return getInterpreter().getExecutable(jobExecutor, jobToken, getImage(), runAs, builtInRegistryAccessToken, isUseTTY());
} else {
return getInterpreter().getExecutable(jobExecutor, jobToken, null, null, isUseTTY());
return getInterpreter().getExecutable(jobExecutor, jobToken, null, null, null, isUseTTY());
}
}

5
server-core/src/main/java/io/onedev/server/buildspec/step/GenerateChecksumStep.java
View File

@ -55,6 +55,11 @@ public class GenerateChecksumStep extends CommandStep {
return "ubuntu:20.04";
}
@Override
public String getRunAs() {
return null;
}
@Override
public boolean isUseTTY() {
return false;

25
server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java
View File

@ -8,6 +8,7 @@ import io.onedev.server.SubscriptionManager;
import io.onedev.server.annotation.ChoiceProvider;
import io.onedev.server.annotation.Editable;
import io.onedev.server.annotation.Interpolative;
import io.onedev.server.annotation.RegEx;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.buildspec.job.EnvVar;
import io.onedev.server.buildspec.param.ParamCombination;
@ -33,6 +34,8 @@ public class RunContainerStep extends Step {
private String image;
private String runAs;
private String args;
private List<EnvVar> envVars = new ArrayList<>();
@ -43,7 +46,7 @@ public class RunContainerStep extends Step {
private String builtInRegistryAccessTokenSecret;
private boolean useTTY;
private boolean useTTY = true;
@Editable(order=100, description="Specify container image to run")
@Interpolative(variableSuggester="suggestVariables")
@ -56,6 +59,18 @@ public class RunContainerStep extends Step {
this.image = image;
}
@Editable(order=150, name="Run As", placeholder = "root", description = "Optionally specify uid:gid to run container as. " +
"<b class='text-warning'>Note:</b> This setting should be left empty if underlying container facility is " +
"rootless or use user namespace remapping")
@RegEx(pattern="\\d+:\\d+", message = "Should be specified in form of <uid>:<gid>")
public String getRunAs() {
return runAs;
}
public void setRunAs(String runAs) {
this.runAs = runAs;
}
@Editable(order=200, name="Arguments", description="Optionally specify container arguments separated by space. " +
"Single argument containing space should be quoted. <b class='text-warning'>Note: </b> do not confuse " +
"this with container options which should be specified in executor setting")
@ -68,7 +83,7 @@ public class RunContainerStep extends Step {
this.args = args;
}
@Editable(order=300, name="Working Directory", description="Optionally specify working directory of the container. "
@Editable(order=300, name="Working Directory", placeholder = "Container default", group="More Settings", description="Optionally specify working directory of the container. "
+ "Leave empty to use default working directory of the container")
@Interpolative(variableSuggester="suggestVariables")
@Nullable
@ -80,7 +95,7 @@ public class RunContainerStep extends Step {
this.workingDir = workingDir;
}
@Editable(order=400, name="Environment Variables", description="Optionally specify environment "
@Editable(order=400, name="Environment Variables", group="More Settings", description="Optionally specify environment "
+ "variables for the container")
public List<EnvVar> getEnvVars() {
return envVars;
@ -89,7 +104,7 @@ public class RunContainerStep extends Step {
public void setEnvVars(List<EnvVar> envVars) {
this.envVars = envVars;
}
@Editable(order=500, group = "More Settings", description="Optionally mount directories or files under job workspace into container")
public List<VolumeMount> getVolumeMounts() {
return volumeMounts;
@ -155,7 +170,7 @@ public class RunContainerStep extends Step {
builtInRegistryAccessToken = build.getJobAuthorizationContext().getSecretValue(getBuiltInRegistryAccessTokenSecret());
else
builtInRegistryAccessToken = null;
return new RunContainerFacade(getImage(), null, getArgs(), envMap, getWorkingDir(), mountMap,
return new RunContainerFacade(getImage(), getRunAs(), getArgs(), envMap, getWorkingDir(), mountMap,
builtInRegistryAccessToken, isUseTTY());
}

5
server-core/src/main/java/io/onedev/server/buildspec/step/SCPCommandStep.java
View File

@ -91,6 +91,11 @@ public class SCPCommandStep extends CommandStep {
return "1dev/ssh-client:1.1.0";
}
@Override
public String getRunAs() {
return null;
}
@Override
public boolean isUseTTY() {
return false;

5
server-core/src/main/java/io/onedev/server/buildspec/step/SSHCommandStep.java
View File

@ -110,6 +110,11 @@ public class SSHCommandStep extends CommandStep {
return "1dev/ssh-client:1.1.0";
}
@Override
public String getRunAs() {
return null;
}
@Override
public boolean isUseTTY() {
return false;

4
server-core/src/main/java/io/onedev/server/buildspec/step/commandinterpreter/DefaultInterpreter.java
View File

@ -30,8 +30,8 @@ public class DefaultInterpreter extends Interpreter {
@Override
public CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, String image,
String builtInRegistryAccessToken, boolean useTTY) {
return new CommandFacade(image, builtInRegistryAccessToken, getCommands(), useTTY);
String runAs, String builtInRegistryAccessToken, boolean useTTY) {
return new CommandFacade(image, runAs, builtInRegistryAccessToken, getCommands(), useTTY);
}
}

15
server-core/src/main/java/io/onedev/server/buildspec/step/commandinterpreter/Interpreter.java
View File

@ -1,17 +1,15 @@
package io.onedev.server.buildspec.step.commandinterpreter;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nullable;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.k8shelper.CommandFacade;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.annotation.Editable;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.List;
@Editable
public abstract class Interpreter implements Serializable {
@ -28,7 +26,8 @@ public abstract class Interpreter implements Serializable {
}
public abstract CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, @Nullable String image,
@Nullable String builtInRegistryAccessToken, boolean useTTY);
@Nullable String runAs, @Nullable String builtInRegistryAccessToken,
boolean useTTY);
static List<InputSuggestion> suggestVariables(String matchWith) {
return BuildSpec.suggestVariables(matchWith, true, false, true);

4
server-core/src/main/java/io/onedev/server/buildspec/step/commandinterpreter/PowerShellInterpreter.java
View File

@ -35,8 +35,8 @@ public class PowerShellInterpreter extends Interpreter {
@Override
public CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, String image,
String builtInRegistryAccessToken, boolean useTTY) {
return new PowerShellFacade(image, builtInRegistryAccessToken, getCommands(), useTTY);
String runAs, String builtInRegistryAccessToken, boolean useTTY) {
return new PowerShellFacade(image, runAs, builtInRegistryAccessToken, getCommands(), useTTY);
}
}

4
server-core/src/main/java/io/onedev/server/buildspec/step/commandinterpreter/ShellInterpreter.java
View File

@ -43,8 +43,8 @@ public class ShellInterpreter extends Interpreter {
@Override
public CommandFacade getExecutable(JobExecutor jobExecutor, String jobToken, String image,
String builtInRegistryAccessToken, boolean useTTY) {
return new ShellFacade(image, builtInRegistryAccessToken, shell, getCommands(), useTTY);
String runAs, String builtInRegistryAccessToken, boolean useTTY) {
return new ShellFacade(image, runAs, builtInRegistryAccessToken, shell, getCommands(), useTTY);
}
}

18
server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
View File

@ -390,7 +390,7 @@ public class DataMigrator {
String content = FileUtils.readFileToString(file, UTF_8);
content = StringUtils.replace(content, "gitplex", "turbodev");
content = StringUtils.replace(content, "GitPlex", "TurboDev");
FileUtils.writeFile(file, content, UTF_8.name());
FileUtils.writeFile(file, content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -425,7 +425,7 @@ public class DataMigrator {
content = StringUtils.replace(content, "turbodev.com", "onedev.io");
content = StringUtils.replace(content, "turbodev", "onedev");
content = StringUtils.replace(content, "TurboDev", "OneDev");
FileUtils.writeFile(file, content, UTF_8.name());
FileUtils.writeFile(file, content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -569,11 +569,11 @@ public class DataMigrator {
FileUtils.deleteFile(file);
} else if (file.getName().startsWith("UserAuthorizations.xml") || file.getName().startsWith("GroupAuthorizations.xml")) {
try {
String content = FileUtils.readFileToString(file, UTF_8.name());
String content = FileUtils.readFileToString(file, UTF_8);
content = StringUtils.replace(content, "ADMIN", "ADMINISTRATION");
content = StringUtils.replace(content, "WRITE", "CODE_WRITE");
content = StringUtils.replace(content, "READ", "CODE_READ");
FileUtils.writeFile(file, content, UTF_8.name());
FileUtils.writeFile(file, content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -1199,7 +1199,7 @@ public class DataMigrator {
File renamedFile = new File(dataDir, file.getName().replace(
"IssueFieldEntitys.xml", "IssueFields.xml"));
FileUtils.writeFile(renamedFile, content, UTF_8.name());
FileUtils.writeFile(renamedFile, content, UTF_8);
}
}
try (InputStream is = getClass().getResourceAsStream("migrate25_roles.xml")) {
@ -1749,12 +1749,12 @@ public class DataMigrator {
for (File file : dataDir.listFiles()) {
if (file.getName().contains(".xml")) {
try {
String content = FileUtils.readFileToString(file, UTF_8.name());
String content = FileUtils.readFileToString(file, UTF_8);
content = StringUtils.replace(content, "io.onedev.server.issue.",
"io.onedev.server.model.support.issue.");
content = StringUtils.replace(content, "io.onedev.server.util.inputspec.",
"io.onedev.server.model.support.inputspec.");
FileUtils.writeFile(file, content, UTF_8.name());
FileUtils.writeFile(file, content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -2016,7 +2016,7 @@ public class DataMigrator {
content = StringUtils.replace(content,
"io.onedev.server.model.support.issue.transitiontrigger.OpenPullRequest",
"io.onedev.server.model.support.issue.transitiontrigger.OpenPullRequestTrigger");
FileUtils.writeFile(file, content, UTF_8.name());
FileUtils.writeFile(file, content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -2071,7 +2071,7 @@ public class DataMigrator {
content = StringUtils.replace(content, "PullRequestBuild", "PullRequestVerification");
FileUtils.deleteFile(file);
String newFileName = StringUtils.replace(file.getName(), "PullRequestBuild", "PullRequestVerification");
FileUtils.writeFile(new File(dataDir, newFileName), content, UTF_8.name());
FileUtils.writeFile(new File(dataDir, newFileName), content, UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}

35
server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
View File

@ -270,7 +270,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
@Editable(order=28000, group="More Settings", description = "Optionally maps a docker image to a different " +
"image. The first matching entry will take effect, or image will remain unchanged if no matching entries " +
"found. For instance a mapping entry with <code>From</code> specified as <code>code.onedev.io/onedev/k8s-helper-linux:(.*)</code>, " +
"found. For instance a mapping entry with <code>From</code> specified as <code>1dev/k8s-helper-linux:(.*)</code>, " +
"and <code>To</code> specified as <code>my-local-registry/k8s-helper-linux:$1</code> will map the " +
"k8s helper image from official docker registry to local registry, with repository and tag unchanged")
public List<ImageMapping> getImageMappings() {
@ -434,7 +434,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
maskedYaml = StringUtils.replace(maskedYaml, secret, SecretInput.MASK);
logger.trace("Creating resource:\n" + maskedYaml);
FileUtils.writeFile(file, resourceYaml, UTF_8.name());
FileUtils.writeFile(file, resourceYaml, UTF_8);
kubectl.addArgs("create", "-f", file.getAbsolutePath(), "-o", "jsonpath={.metadata.name}");
kubectl.execute(new LineConsumer() {
@ -771,6 +771,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
containerSpec.put("args", argList);
}
containerSpec.put("env", envs);
setupSecurityContext(containerSpec, jobService.getRunAs());
podSpec.put("containers", Lists.<Object>newArrayList(containerSpec));
if (imagePullSecretName != null)
@ -893,6 +894,16 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
return map;
}
private void setupSecurityContext(Map<Object, Object> containerSpec, @Nullable String runAs) {
if (runAs != null) {
var securityContext = new HashMap<>();
var fields = Splitter.on(':').trimResults().splitToList(runAs);
securityContext.put("runAsUser", fields.get(0));
securityContext.put("runAsGroup", fields.get(1));
containerSpec.put("securityContext", securityContext);
}
}
private void execute(TaskLogger jobLogger, Object executionContext) {
jobLogger.log("Checking cluster access...");
JobContext jobContext;
@ -969,43 +980,28 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
String containerBuildHome;
String containerCommandDir;
String containerAuthInfoDir;
String containerTrustCertsDir;
String containerWorkspace;
if (osInfo.isWindows()) {
containerBuildHome = "C:\\onedev-build";
containerWorkspace = containerBuildHome + "\\workspace";
containerCommandDir = containerBuildHome + "\\command";
containerAuthInfoDir = "C:\\Users\\ContainerAdministrator\\auth-info";
containerTrustCertsDir = containerBuildHome + "\\trust-certs";
} else {
containerBuildHome = "/onedev-build";
containerWorkspace = containerBuildHome +"/workspace";
containerCommandDir = containerBuildHome + "/command";
containerAuthInfoDir = "/root/auth-info";
containerTrustCertsDir = containerBuildHome + "/trust-certs";
}
Map<String, String> buildHomeMount = newLinkedHashMap(
"name", "build-home",
"mountPath", containerBuildHome);
Map<String, String> authInfoMount = newLinkedHashMap(
"name", "build-home",
"mountPath", containerAuthInfoDir,
"subPath", "auth-info");
// Windows nanoserver default user is ContainerUser
Map<String, String> authInfoMount2 = newLinkedHashMap(
"name", "build-home",
"mountPath", "C:\\Users\\ContainerUser\\auth-info",
"subPath", "auth-info");
Map<String, String> trustCertsMount = newLinkedHashMap(
"name", "trust-certs",
"mountPath", containerTrustCertsDir);
var commonVolumeMounts = newArrayList(buildHomeMount, authInfoMount);
if (osInfo.isWindows())
commonVolumeMounts.add(authInfoMount2);
var commonVolumeMounts = newArrayList(buildHomeMount);
if (trustCertsConfigMapName != null)
commonVolumeMounts.add(trustCertsMount);
@ -1014,7 +1010,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
entryFacade = new CompositeFacade(jobContext.getActions());
} else {
List<Action> actions = new ArrayList<>();
CommandFacade facade = new CommandFacade((String) executionContext, null,
CommandFacade facade = new CommandFacade((String) executionContext, null, null,
"this does not matter", false);
actions.add(new Action("test", facade, ExecuteCondition.ALWAYS));
entryFacade = new CompositeFacade(actions);
@ -1073,6 +1069,7 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
volumeMounts.addAll(commonVolumeMounts);
stepContainerSpec.put("volumeMounts", SerializationUtils.clone(volumeMounts));
stepContainerSpec.put("env", SerializationUtils.clone(commonEnvs));
setupSecurityContext(stepContainerSpec, execution.getRunAs());
} else if (facade instanceof BuildImageFacade) {
throw new ExplicitException("This step can only be executed by server docker executor or " +
"remote docker executor. Use kaniko step instead to build image in kubernetes cluster");

85
server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
View File

@ -12,7 +12,6 @@ import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.loader.AppLoader;
import io.onedev.commons.utils.*;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.ExecutionResult;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.k8shelper.*;
import io.onedev.server.OneDev;
@ -41,8 +40,9 @@ import javax.validation.constraints.NotEmpty;
import java.io.File;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicBoolean;
import static io.onedev.agent.DockerExecutorUtils.changeOwner;
import static io.onedev.agent.DockerExecutorUtils.*;
import static io.onedev.k8shelper.KubernetesHelper.*;
import static java.util.stream.Collectors.toList;
@ -281,11 +281,12 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
}
File hostWorkspace = new File(hostBuildHome, "workspace");
File hostUserHome = new File(hostBuildHome, "user");
FileUtils.createDir(hostWorkspace);
FileUtils.createDir(hostUserHome);
var cacheHelper = new ServerCacheHelper(hostBuildHome, jobContext, jobLogger);
AtomicReference<File> hostAuthInfoDir = new AtomicReference<>(null);
try {
jobLogger.log("Copying job dependencies...");
getJobManager().copyDependencies(jobContext, hostWorkspace);
@ -303,14 +304,15 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
containerTrustCerts = "/onedev-build/trust-certs.pem";
}
var ownerChanged = new AtomicBoolean(false);
getJobManager().reportJobWorkspace(jobContext, containerWorkspace);
CompositeFacade entryFacade = new CompositeFacade(jobContext.getActions());
boolean successful = entryFacade.execute(new LeafHandler() {
private int runStepContainer(Commandline docker, String image, @Nullable String entrypoint,
List<String> options, List<String> arguments, Map<String, String> environments,
@Nullable String workingDir, Map<String, String> volumeMounts,
List<Integer> position, boolean useTTY) {
private int runStepContainer(Commandline docker, String image, @Nullable String runAs,
@Nullable String entrypoint, List<String> arguments,
Map<String, String> environments, @Nullable String workingDir,
Map<String, String> volumeMounts, List<Integer> position, boolean useTTY) {
image = mapImage(image);
containerName = network + "-step-" + stringifyStepPosition(position);
try {
@ -318,6 +320,11 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
docker.clearArgs();
docker.addArgs("run", "--name=" + containerName, "--network=" + network);
if (runAs != null)
docker.addArgs("--user", runAs);
else if (!SystemUtils.IS_OS_WINDOWS)
docker.addArgs("--user", "0:0");
if (getCpuLimit() != null)
docker.addArgs("--cpus", getCpuLimit());
if (getMemoryLimit() != null)
@ -355,16 +362,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
}
}
if (hostAuthInfoDir.get() != null) {
String hostPath = getHostPath(hostAuthInfoDir.get().getAbsolutePath());
if (SystemUtils.IS_OS_WINDOWS) {
docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerAdministrator\\auth-info");
docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerUser\\auth-info");
} else {
docker.addArgs("-v", hostPath + ":/root/auth-info");
}
}
for (Map.Entry<String, String> entry : environments.entrySet())
docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());
@ -378,13 +375,12 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
if (useProcessIsolation)
docker.addArgs("--isolation=process");
docker.addArgs(options.toArray(new String[0]));
docker.addArgs(image);
docker.addArgs(arguments.toArray(new String[0]));
docker.processKiller(newDockerKiller(newDocker(), containerName, jobLogger));
ExecutionResult result = docker.execute(ExecutorUtils.newInfoLogger(jobLogger),
var result = docker.execute(ExecutorUtils.newInfoLogger(jobLogger),
ExecutorUtils.newWarningLogger(jobLogger), null);
return result.getReturnCode();
} finally {
@ -398,8 +394,13 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
try {
String stepNames = entryFacade.getNamesAsString(position);
jobLogger.notice("Running step \"" + stepNames + "\"...");
long time = System.currentTimeMillis();
if (ownerChanged.get() && !Bootstrap.isInDocker()) {
changeOwner(hostBuildHome, getOwner(), newDocker(), false);
ownerChanged.set(false);
}
if (facade instanceof CommandFacade) {
CommandFacade commandFacade = (CommandFacade) facade;
@ -408,16 +409,18 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
throw new ExplicitException("This step can only be executed by server shell "
+ "executor or remote shell executor");
}
Commandline entrypoint = DockerExecutorUtils.getEntrypoint(
hostBuildHome, commandFacade, osInfo, hostAuthInfoDir.get() != null);
var docker = newDocker();
Commandline entrypoint = getEntrypoint(hostBuildHome, commandFacade, osInfo);
var builtInRegistryLogin = new BuiltInRegistryLogin(serverUrl,
jobContext.getJobToken(), commandFacade.getBuiltInRegistryAccessToken());
var docker = newDocker();
if (changeOwner(hostBuildHome, execution.getRunAs(), docker, Bootstrap.isInDocker()))
ownerChanged.set(true);
docker.clearArgs();
int exitCode = callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
return runStepContainer(docker, execution.getImage(), entrypoint.executable(),
new ArrayList<>(), entrypoint.arguments(), new HashMap<>(), null,
return runStepContainer(docker, execution.getImage(), execution.getRunAs(),
entrypoint.executable(), entrypoint.arguments(), new HashMap<>(), null,
new HashMap<>(), position, commandFacade.isUseTTY());
});
@ -447,20 +450,20 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
} else if (facade instanceof RunContainerFacade) {
RunContainerFacade runContainerFacade = (RunContainerFacade) facade;
OsContainer container = runContainerFacade.getContainer(osInfo);
List<String> options;
if (container.getOpts() != null)
options = Splitter.on(" ").trimResults().omitEmptyStrings().splitToList(container.getOpts());
else
options = new ArrayList<>();
List<String> arguments = new ArrayList<>();
if (container.getArgs() != null)
arguments.addAll(Arrays.asList(StringUtils.parseQuoteTokens(container.getArgs())));
var docker = newDocker();
var builtInRegistryLogin =new BuiltInRegistryLogin(serverUrl,
jobContext.getJobToken(), runContainerFacade.getBuiltInRegistryAccessToken());
var docker = newDocker();
if (changeOwner(hostBuildHome, container.getRunAs(), docker, Bootstrap.isInDocker()))
ownerChanged.set(true);
docker.clearArgs();
int exitCode = callWithDockerAuth(docker, getRegistryLoginFacades(), builtInRegistryLogin, () -> {
return runStepContainer(docker, container.getImage(), null, options, arguments,
container.getEnvMap(), container.getWorkingDir(), container.getVolumeMounts(),
return runStepContainer(docker, container.getImage(),container.getRunAs(), null,
arguments, container.getEnvMap(), container.getWorkingDir(), container.getVolumeMounts(),
position, runContainerFacade.isUseTTY());
});
if (exitCode != 0) {
@ -475,9 +478,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
Commandline git = new Commandline(AppLoader.getInstance(GitLocation.class).getExecutable());
if (hostAuthInfoDir.get() == null)
hostAuthInfoDir.set(FileUtils.createTempDir());
git.environments().put("HOME", hostAuthInfoDir.get().getAbsolutePath());
git.environments().put("HOME", hostUserHome.getAbsolutePath());
checkoutFacade.setupWorkingDir(git, hostWorkspace);
@ -493,7 +494,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
ExecutorUtils.newWarningLogger(jobLogger));
CloneInfo cloneInfo = checkoutFacade.getCloneInfo();
cloneInfo.writeAuthData(hostAuthInfoDir.get(), git, true,
cloneInfo.writeAuthData(hostUserHome, git, true,
ExecutorUtils.newInfoLogger(jobLogger),
ExecutorUtils.newWarningLogger(jobLogger));
@ -560,8 +561,6 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
if (SystemUtils.IS_OS_WINDOWS) {
FileUtils.deleteDir(hostWorkspace);
}
if (hostAuthInfoDir.get() != null)
FileUtils.deleteDir(hostAuthInfoDir.get());
}
} finally {
deleteNetwork(newDocker(), network, jobLogger);

1
server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
View File

@ -39,6 +39,7 @@ import static io.onedev.agent.ExecutorUtils.newInfoLogger;
import static io.onedev.agent.ExecutorUtils.newWarningLogger;
import static io.onedev.agent.ShellExecutorUtils.testCommands;
import static io.onedev.k8shelper.KubernetesHelper.*;
import static java.nio.charset.StandardCharsets.UTF_8;
@Editable(order=ServerShellExecutor.ORDER, name="Server Shell Executor", description="" +
"This executor runs build jobs with OneDev server's shell facility.<br>" +

5
server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java
View File

@ -20,6 +20,7 @@ import io.onedev.server.annotation.Editable;
import io.onedev.server.annotation.Interpolative;
import static io.onedev.commons.utils.LockUtils.write;
import static java.nio.charset.StandardCharsets.UTF_8;
@Editable(order=1200, group=StepGroup.PUBLISH, name="Pull Request Markdown Report",
description="This report will be displayed in pull request overview page if build is triggered by pull request")
@ -61,9 +62,9 @@ public class PublishPullRequestMarkdownReportStep extends PublishReportStep {
File file = new File(workspace, getFile());
if (file.exists()) {
File reportDir = new File(build.getDir(), CATEGORY + "/" + getReportName());
String markdown = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
String markdown = FileUtils.readFileToString(file, UTF_8);
FileUtils.createDir(reportDir);
FileUtils.writeFile(new File(reportDir, CONTENT), markdown, StandardCharsets.UTF_8.name());
FileUtils.writeFile(new File(reportDir, CONTENT), markdown, UTF_8);
OneDev.getInstance(ProjectManager.class).directoryModified(
build.getProject().getId(), reportDir.getParentFile());
} else {