diff --git a/pom.xml b/pom.xml index 567c92fb92..29133fa93e 100644 --- a/pom.xml +++ b/pom.xml @@ -637,7 +637,7 @@ 3.0.4 - 2.1.8 + 2.1.9 2.0.9 1.4.14 4.7.2 diff --git a/server-plugin/server-plugin-buildspec-python/pom.xml b/server-plugin/server-plugin-buildspec-python/pom.xml index bb5ed56469..b9d7b1872c 100644 --- a/server-plugin/server-plugin-buildspec-python/pom.xml +++ b/server-plugin/server-plugin-buildspec-python/pom.xml @@ -7,6 +7,13 @@ server-plugin 11.0.9 + + + io.onedev + server-plugin-report-junit + ${project.version} + + io.onedev.server.plugin.buildspec.python.PythonModule diff --git a/server-plugin/server-plugin-buildspec-python/src/main/java/io/onedev/server/plugin/buildspec/python/PythonJobSuggestion.java b/server-plugin/server-plugin-buildspec-python/src/main/java/io/onedev/server/plugin/buildspec/python/PythonJobSuggestion.java index 3380d4ae6e..b2c7722445 100644 --- a/server-plugin/server-plugin-buildspec-python/src/main/java/io/onedev/server/plugin/buildspec/python/PythonJobSuggestion.java +++ b/server-plugin/server-plugin-buildspec-python/src/main/java/io/onedev/server/plugin/buildspec/python/PythonJobSuggestion.java @@ -1,6 +1,7 @@ package io.onedev.server.plugin.buildspec.python; import com.google.common.collect.Lists; +import io.onedev.k8shelper.ExecuteCondition; import io.onedev.server.buildspec.job.Job; import io.onedev.server.buildspec.job.JobSuggestion; import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger; @@ -10,14 +11,26 @@ import io.onedev.server.buildspec.step.commandinterpreter.ShellInterpreter; import io.onedev.server.git.Blob; import io.onedev.server.git.BlobIdent; import io.onedev.server.model.Project; +import io.onedev.server.plugin.report.junit.PublishJUnitReportStep; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; +import org.yaml.snakeyaml.Yaml; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; public class PythonJobSuggestion implements JobSuggestion { + private PublishJUnitReportStep newUnitTestReportPublishStep() { + var publishUnitTestReport = new PublishJUnitReportStep(); + publishUnitTestReport.setName("publish unit test report"); + publishUnitTestReport.setReportName("Unit Test"); + publishUnitTestReport.setFilePatterns("test-result.xml"); + publishUnitTestReport.setCondition(ExecuteCondition.ALWAYS); + return publishUnitTestReport; + } + @Override public Collection suggestJobs(Project project, ObjectId commitId) { Collection jobs = new ArrayList<>(); @@ -45,10 +58,8 @@ public class PythonJobSuggestion implements JobSuggestion { CommandStep detectBuildVersion = new CommandStep(); detectBuildVersion.setName("detect build version"); - detectBuildVersion.setImage("alpine"); - detectBuildVersion.getInterpreter().setCommands("" + - "apk add yq\n" + - "yq '.tool.poetry.version' pyproject.toml > buildVersion\n"); + detectBuildVersion.setImage("1dev/yq:1.0.0"); + detectBuildVersion.getInterpreter().setCommands("yq '.tool.poetry.version' pyproject.toml > buildVersion"); job.getSteps().add(detectBuildVersion); SetBuildVersionStep setBuildVersion = new SetBuildVersionStep(); @@ -58,18 +69,17 @@ public class PythonJobSuggestion implements JobSuggestion { CommandStep test = new CommandStep(); test.setName("test"); - test.setImage("python"); - String commands = "" + - "curl -sSL https://install.python-poetry.org | python\n" + - "/root/.local/bin/poetry install\n"; + test.setImage("1dev/poetry:1.0.1"); + String commands = "poetry install\n"; - if (blob.getText().getContent().contains("pytest")) - commands += "/root/.local/bin/poetry run pytest"; - else - commands += "/root/.local/bin/poetry run python -m unittest"; - test.getInterpreter().setCommands(commands); - - job.getSteps().add(test); + if (blob.getText().getContent().contains("\"pytest\"")) { + test.getInterpreter().setCommands(commands + "poetry run pytest --junitxml=./test-result.xml"); + job.getSteps().add(test); + job.getSteps().add(newUnitTestReportPublishStep()); + } else { + test.getInterpreter().setCommands(commands + "poetry run python -m unittest"); + job.getSteps().add(test); + } job.getTriggers().add(new BranchUpdateTrigger()); job.getTriggers().add(new PullRequestUpdateTrigger()); @@ -106,17 +116,68 @@ public class PythonJobSuggestion implements JobSuggestion { "source .venv/bin/activate\n" + "pip install -r requirements.txt\n"; - if (blob.getText().getContent().contains("pytest")) - commands += "pytest"; - else - commands += "python -m unittest"; - interpreter.setCommands(commands); - job.getSteps().add(test); + var blobContent = blob.getText().getContent(); + if (blobContent.contains("pytest==")) { + interpreter.setCommands(commands + "pytest --junitxml=./test-result.xml"); + job.getSteps().add(test); + job.getSteps().add(newUnitTestReportPublishStep()); + } else if (blobContent.contains("Django==")) { + interpreter.setCommands(commands + "python manage.py test"); + job.getSteps().add(test); + } else { + interpreter.setCommands(commands + "python -m unittest"); + job.getSteps().add(test); + } job.getTriggers().add(new BranchUpdateTrigger()); job.getTriggers().add(new PullRequestUpdateTrigger()); jobs.add(job); + } else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "environment.yml", FileMode.TYPE_FILE), false)) != null) { + Job job = new Job(); + job.setName("python ci"); + + CheckoutStep checkout = new CheckoutStep(); + checkout.setName("checkout code"); + job.getSteps().add(checkout); + + var generateChecksum = new GenerateChecksumStep(); + generateChecksum.setName("generate environment checksum"); + generateChecksum.setFiles("environment.yml"); + generateChecksum.setTargetFile("checksum"); + job.getSteps().add(generateChecksum); + + var setupCache = new SetupCacheStep(); + setupCache.setName("set up environment cache"); + setupCache.setKey("conda_cache_@file:checksum@"); + setupCache.setPaths(Lists.newArrayList("/root/miniconda3/envs")); + setupCache.getLoadKeys().add("conda_cache"); + job.getSteps().add(setupCache); + + var blobContent = blob.getText().getContent(); + Map environments = new Yaml().load(blobContent); + CommandStep test = new CommandStep(); + test.setName("test"); + test.setImage("1dev/conda:1.0.4"); + test.setInterpreter(new ShellInterpreter()); + String commands = "" + + "source /root/.bashrc\n" + + "conda env update\n" + + "conda activate " + environments.get("name") + "\n"; + + if (blobContent.contains("pytest")) { + test.getInterpreter().setCommands(commands + "pytest --junitxml=./test-result.xml"); + job.getSteps().add(test); + job.getSteps().add(newUnitTestReportPublishStep()); + } else { + test.getInterpreter().setCommands(commands + "python -m unittest"); + job.getSteps().add(test); + } + + job.getTriggers().add(new BranchUpdateTrigger()); + job.getTriggers().add(new PullRequestUpdateTrigger()); + + jobs.add(job); } return jobs; } diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java index da65e9ec41..7a9d5977ee 100644 --- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java +++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java @@ -571,7 +571,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw }, new ArrayList<>()); - cacheHelper.buildFinished(); + cacheHelper.buildFinished(successful); return successful; } finally { diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java index 4f7c4f300c..c504764412 100644 --- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java +++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java @@ -244,7 +244,7 @@ public class ServerShellExecutor extends JobExecutor implements Testable()); - cacheHelper.buildFinished(); + cacheHelper.buildFinished(successful); return successful; } finally {