diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java index f4b3c9deee..52bca3a5c0 100644 --- a/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java +++ b/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java @@ -115,7 +115,7 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemen } private boolean collect(Project project) { - logger.debug("Collecting build info in project '{}'...", project); + logger.debug("Collecting build info (project: {})...", project); Environment env = getEnv(project.getId().toString()); Store defaultStore = getStore(env, DEFAULT_STORE); @@ -164,7 +164,7 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemen }); } - + logger.debug("Collected build info (project: {})", project); return unprocessedBuilds.size() == BATCH_SIZE; } diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java index d7c33d38fc..772214928e 100644 --- a/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java +++ b/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java @@ -140,7 +140,7 @@ public class DefaultCodeCommentRelationInfoManager extends AbstractEnvironmentMa } private boolean collect(Project project) { - logger.debug("Collecting code comment relation info in project '{}'...", project); + logger.debug("Collecting code comment relation info (project: {})...", project); Environment env = getEnv(project.getId().toString()); Store defaultStore = getStore(env, DEFAULT_STORE); @@ -264,6 +264,7 @@ public class DefaultCodeCommentRelationInfoManager extends AbstractEnvironmentMa }); } } + logger.debug("Collected code comment relation info (project: {})", project); return unprocessedPullRequestUpdates.size() == BATCH_SIZE || unprocessedCodeComments.size() == BATCH_SIZE; } diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java index 04eb61e3c3..2752894336 100644 --- a/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java +++ b/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java @@ -314,6 +314,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme } private void doCollect(Project project, ObjectId commitId, String refName) { + logger.debug("Collecting commit information (project: {}, ref: {})...", refName, project.getName()); + Environment env = getEnv(project.getId().toString()); Store defaultStore = getStore(env, DEFAULT_STORE); Store commitsStore = getStore(env, COMMITS_STORE); @@ -762,6 +764,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme } } } + logger.debug("Collected commit information (project: {}, ref: {})", project.getName(), refName); } private void updateContribution(Map contributions, int key, GitCommit commit) { @@ -991,11 +994,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme collectingWorks.add((CollectingWork)work); Collections.sort(collectingWorks, new CommitTimeComparator()); - for (CollectingWork work: collectingWorks) { - logger.debug("Collecting commit information up to ref '{}' in project '{}'...", - work.getRefName(), project.getName()); + for (CollectingWork work: collectingWorks) doCollect(project, work.getCommit().copy(), work.getRefName()); - } } }); diff --git a/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.java b/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.java new file mode 100644 index 0000000000..644d810490 --- /dev/null +++ b/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.java @@ -0,0 +1,53 @@ +package io.onedev.server.model.support; + +import java.io.Serializable; + +import org.hibernate.validator.constraints.NotEmpty; + +import io.onedev.server.web.editable.annotation.Editable; +import io.onedev.server.web.editable.annotation.NameOfEmptyValue; +import io.onedev.server.web.editable.annotation.Password; + +@Editable +public class RegistryLogin implements Serializable { + + private static final long serialVersionUID = 1L; + + private String registryUrl; + + private String userName; + + private String password; + + @Editable(order=100, description="Specify registry url. Leave empty for official registry") + @NameOfEmptyValue("Default Registry") + public String getRegistryUrl() { + return registryUrl; + } + + public void setRegistryUrl(String registryUrl) { + this.registryUrl = registryUrl; + } + + @Editable(order=200) + @NotEmpty + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + @Editable(order=300) + @NotEmpty + @Password + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + +} \ No newline at end of file diff --git a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java index 060f281372..4b9b0498e4 100644 --- a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java +++ b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Base64; +import java.util.Base64.Encoder; import java.util.Collection; import java.util.Iterator; import java.util.LinkedHashMap; @@ -21,10 +22,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -42,6 +43,7 @@ import io.onedev.server.ci.job.CacheSpec; import io.onedev.server.ci.job.JobContext; import io.onedev.server.entitymanager.SettingManager; import io.onedev.server.model.support.JobExecutor; +import io.onedev.server.model.support.RegistryLogin; import io.onedev.server.plugin.kubernetes.KubernetesExecutor.TestData; import io.onedev.server.util.JobLogger; import io.onedev.server.util.inputspec.SecretInput; @@ -71,7 +73,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable nodeSelector = new ArrayList<>(); - private String imagePullSecrets; + private List registryLogins = new ArrayList<>(); private String serviceAccount; @@ -125,16 +127,14 @@ public class KubernetesExecutor extends JobExecutor implements Testablekubernetes " - + "documentation on how to set up image pull secrets") - public String getImagePullSecrets() { - return imagePullSecrets; + @Editable(order=22000, group="More Settings", description="Specify login information of docker registries if necessary. These " + + "logins will be used to create image pull secrets of the job pods") + public List getRegistryLogins() { + return registryLogins; } - public void setImagePullSecrets(String imagePullSecrets) { - this.imagePullSecrets = imagePullSecrets; + public void setRegistryLogins(List registryLogins) { + this.registryLogins = registryLogins; } @Editable(order=23000, group="More Settings", description="Optionally specify a service account in above namespace to run the job " @@ -200,6 +200,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable getImagePullSecretsData() { - List data = new ArrayList<>(); - if (getImagePullSecrets() != null) { - for (String imagePullSecret: Splitter.on(" ").trimResults().omitEmptyStrings().split(getImagePullSecrets())) - data.add(Maps.newLinkedHashMap("name", imagePullSecret)); - } - return data; - } - @Nullable private Map getAffinity(@Nullable JobContext jobContext) { Map nodeAffinity = new LinkedHashMap<>(); @@ -408,13 +400,41 @@ public class KubernetesExecutor extends JobExecutor implements Testable auths = new LinkedHashMap<>(); + for (RegistryLogin login: getRegistryLogins()) { + String auth = login.getUserName() + ":" + login.getPassword(); + String registryUrl = login.getRegistryUrl(); + if (registryUrl == null) + registryUrl = "https://index.docker.io/v1/"; + auths.put(registryUrl, Maps.newLinkedHashMap( + "auth", encoder.encodeToString(auth.getBytes(Charsets.UTF_8)))); + } + try { + String dockerConfig = new ObjectMapper().writeValueAsString(Maps.newLinkedHashMap("auths", auths)); + return createSecret(Maps.newLinkedHashMap(".dockerconfigjson", dockerConfig), "kubernetes.io/dockerconfigjson", logger); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } else { + return null; + } + } private void execute(String dockerImage, String jobToken, JobLogger logger, @Nullable JobContext jobContext) { createNamespaceIfNotExist(logger); - Map secrets = Maps.newLinkedHashMap(KubernetesHelper.ENV_JOB_TOKEN, jobToken); - String secretName = createSecret(secrets, logger); + String jobSecretName = null; + String imagePullSecretName = null; try { + Map jobSecrets = Maps.newLinkedHashMap(KubernetesHelper.ENV_JOB_TOKEN, jobToken); + jobSecretName = createSecret(jobSecrets, null, logger); + imagePullSecretName = createImagePullSecret(logger); + String osName = getOSName(logger); Map podSpec = new LinkedHashMap<>(); @@ -460,7 +480,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable sidecarArgs = Lists.newArrayList("-classpath", k8sHelperClassPath, "io.onedev.k8shelper.SideCar"); List initArgs = Lists.newArrayList("-classpath", k8sHelperClassPath, "io.onedev.k8shelper.Init"); @@ -491,9 +511,8 @@ public class KubernetesExecutor extends JobExecutor implements Testable imagePullSecretsData = getImagePullSecretsData(); - if (!imagePullSecretsData.isEmpty()) - podSpec.put("imagePullSecrets", imagePullSecretsData); + if (imagePullSecretName != null) + podSpec.put("imagePullSecrets", Lists.newArrayList(Maps.newLinkedHashMap("name", imagePullSecretName))); if (getServiceAccount() != null) podSpec.put("serviceAccountName", getServiceAccount()); podSpec.put("restartPolicy", "Never"); @@ -642,7 +661,10 @@ public class KubernetesExecutor extends JobExecutor implements Testable secrets, JobLogger logger) { + private String createSecret(Map secrets, @Nullable String type, JobLogger logger) { Map encodedSecrets = new LinkedHashMap<>(); for (Map.Entry entry: secrets.entrySet()) encodedSecrets.put(entry.getKey(), Base64.getEncoder().encodeToString(entry.getValue().getBytes(Charsets.UTF_8))); @@ -805,6 +827,8 @@ public class KubernetesExecutor extends JobExecutor implements Testable