archive-gh-me/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-08 18:26:30 +00:00
Code Issues Projects Releases Wiki Activity

Cmake and Bazel support

Browse Source 
feat: Able to suggest ci job for CMake based projects (OD-2097)
feat: Able to suggest ci job for Bazel based projects (OD-2098)
This commit is contained in:
Robin Shen 2024-09-23 15:37:58 +08:00
parent 31d58208cf
commit 516136aff2
13 changed files with 349 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -636,8 +636,8 @@
</repository>
</repositories>
<properties>
<commons.version>3.0.5</commons.version>
<agent.version>2.1.10</agent.version>
<commons.version>3.0.7</commons.version>
<agent.version>2.2.1</agent.version>
<slf4j.version>2.0.9</slf4j.version>
<logback.version>1.4.14</logback.version>
<antlr.version>4.7.2</antlr.version>

2
server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
View File

@ -85,7 +85,7 @@ public class SetupCacheStep extends Step {
this.paths = paths;
}
@Editable(order=400, description = "Specify cache upload strategy. " +
@Editable(order=400, description = "Specify cache upload strategy after build successful. " +
"<var>Upload If Not Hit</var> means to upload when cache is not found with " +
"cache key (not load keys), and <var>Upload If Changed</var> means to upload if some files " +
"in cache path are changed")

2
server-plugin/pom.xml
View File

@ -24,6 +24,8 @@
<module>server-plugin-buildspec-gradle</module>
<module>server-plugin-buildspec-node</module>
<module>server-plugin-buildspec-python</module>
<module>server-plugin-buildspec-cmake</module>
<module>server-plugin-buildspec-bazel</module>
<module>server-plugin-authenticator-ldap</module>
<module>server-plugin-mailservice-smtpimap</module>
<module>server-plugin-mailservice-office365</module>

13
server-plugin/server-plugin-buildspec-bazel/pom.xml Normal file
View File

@ -0,0 +1,13 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>server-plugin-buildspec-bazel</artifactId>
<parent>
<groupId>io.onedev</groupId>
<artifactId>server-plugin</artifactId>
<version>11.1.3</version>
</parent>
<properties>
<moduleClass>io.onedev.server.plugin.buildspec.bazel.BazelModule</moduleClass>
</properties>
</project>

72
server-plugin/server-plugin-buildspec-bazel/src/main/java/io/onedev/server/plugin/buildspec/bazel/BazelJobSuggestion.java Normal file
View File

@ -0,0 +1,72 @@
package io.onedev.server.plugin.buildspec.bazel;
import com.google.common.collect.Lists;
import io.onedev.server.buildspec.job.Job;
import io.onedev.server.buildspec.job.JobSuggestion;
import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
import io.onedev.server.buildspec.job.trigger.PullRequestUpdateTrigger;
import io.onedev.server.buildspec.step.CheckoutStep;
import io.onedev.server.buildspec.step.CommandStep;
import io.onedev.server.buildspec.step.GenerateChecksumStep;
import io.onedev.server.buildspec.step.SetupCacheStep;
import io.onedev.server.git.BlobIdent;
import io.onedev.server.model.Project;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
public class BazelJobSuggestion implements JobSuggestion {
private GenerateChecksumStep newChecksumGenerateStep(String name, String files) {
var generateChecksum = new GenerateChecksumStep();
generateChecksum.setName(name);
generateChecksum.setFiles(files);
generateChecksum.setTargetFile("checksum");
return generateChecksum;
}
private Job newJob() {
Job job = new Job();
job.setName("bazel ci");
return job;
}
private void addCommonJobsAndTriggers(Job job) {
var checkout = new CheckoutStep();
checkout.setName("checkout code");
job.getSteps().add(0, checkout);
job.getTriggers().add(new BranchUpdateTrigger());
job.getTriggers().add(new PullRequestUpdateTrigger());
}
@Override
public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
List<Job> jobs = new ArrayList<>();
if (project.getBlob(new BlobIdent(commitId.name(), "MODULE.bazel", FileMode.TYPE_FILE), false) != null) {
Job job = newJob();
job.getSteps().add(newChecksumGenerateStep("generate bazel checksum", ".bazelversion MODULE.bazel MODULE.bazel.lock **/BUILD **/BUILD.bazel"));
var setupCache = new SetupCacheStep();
setupCache.setName("set up bazel cache");
setupCache.setKey("bazel_cache_@file:checksum@");
setupCache.setPaths(Lists.newArrayList("/root/.cache/bazelisk", "/root/.cache/bazel"));
setupCache.getLoadKeys().add("bazel_cache");
job.getSteps().add(setupCache);
CommandStep testAndCheck = new CommandStep();
testAndCheck.setName("build and test");
testAndCheck.setImage("1dev/bazelisk:1.0.2");
testAndCheck.getInterpreter().setCommands("" +
"set -e\n" +
"bazelisk build //...\n" +
"bazelisk test //...");
job.getSteps().add(testAndCheck);
addCommonJobsAndTriggers(job);
jobs.add(job);
}
return jobs;
}
}

20
server-plugin/server-plugin-buildspec-bazel/src/main/java/io/onedev/server/plugin/buildspec/bazel/BazelModule.java Normal file
View File

@ -0,0 +1,20 @@
package io.onedev.server.plugin.buildspec.bazel;
import io.onedev.commons.loader.AbstractPluginModule;
import io.onedev.server.buildspec.job.JobSuggestion;
/**
* NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
*
*/
public class BazelModule extends AbstractPluginModule {
@Override
protected void configure() {
super.configure();
// put your guice bindings here
contribute(JobSuggestion.class, BazelJobSuggestion.class);
}
}

25
server-plugin/server-plugin-buildspec-cmake/pom.xml Normal file
View File

@ -0,0 +1,25 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>server-plugin-buildspec-cmake</artifactId>
<parent>
<groupId>io.onedev</groupId>
<artifactId>server-plugin</artifactId>
<version>11.1.3</version>
</parent>
<dependencies>
<dependency>
<groupId>io.onedev</groupId>
<artifactId>server-plugin-report-cobertura</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
<artifactId>server-plugin-report-cppcheck</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<properties>
<moduleClass>io.onedev.server.plugin.buildspec.cmake.CmakeModule</moduleClass>
</properties>
</project>

143
server-plugin/server-plugin-buildspec-cmake/src/main/java/io/onedev/server/plugin/buildspec/cmake/CmakeJobSuggestion.java Normal file
View File

@ -0,0 +1,143 @@
package io.onedev.server.plugin.buildspec.cmake;
import com.google.common.collect.Lists;
import io.onedev.k8shelper.ExecuteCondition;
import io.onedev.server.buildspec.job.Job;
import io.onedev.server.buildspec.job.JobSuggestion;
import io.onedev.server.buildspec.job.trigger.BranchUpdateTrigger;
import io.onedev.server.buildspec.job.trigger.PullRequestUpdateTrigger;
import io.onedev.server.buildspec.step.CheckoutStep;
import io.onedev.server.buildspec.step.CommandStep;
import io.onedev.server.buildspec.step.GenerateChecksumStep;
import io.onedev.server.buildspec.step.SetupCacheStep;
import io.onedev.server.git.BlobIdent;
import io.onedev.server.model.Project;
import io.onedev.server.plugin.report.cobertura.PublishCoberturaReportStep;
import io.onedev.server.plugin.report.coverage.PublishCoverageReportStep;
import io.onedev.server.plugin.report.cppcheck.PublishCppcheckReportStep;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
public class CmakeJobSuggestion implements JobSuggestion {
private static final Logger logger = LoggerFactory.getLogger(CmakeJobSuggestion.class);
private GenerateChecksumStep newChecksumGenerateStep(String files) {
var generateChecksum = new GenerateChecksumStep();
generateChecksum.setName("generate dependency checksum");
generateChecksum.setFiles(files);
generateChecksum.setTargetFile("checksum");
return generateChecksum;
}
private SetupCacheStep newVcpkgCacheSetupStep() {
var setupCache = new SetupCacheStep();
setupCache.setName("set up dependency cache");
setupCache.setKey("vcpkg_cache_@file:checksum@");
setupCache.setPaths(Lists.newArrayList("/root/.cache/vcpkg"));
setupCache.getLoadKeys().add("vcpkg_cache");
return setupCache;
}
private SetupCacheStep newFetchContentCacheSetupStep() {
var setupCache = new SetupCacheStep();
setupCache.setName("set up dependency cache");
setupCache.setKey("fetchcontent_cache_@file:checksum@");
setupCache.setPaths(Lists.newArrayList("build/.deps"));
setupCache.getLoadKeys().add("fetchcontent_cache");
return setupCache;
}
private SetupCacheStep newConanCacheSetupStep() {
var setupCache = new SetupCacheStep();
setupCache.setName("set up dependency cache");
setupCache.setKey("conan_cache_@file:checksum@");
setupCache.setPaths(Lists.newArrayList("/root/.conan2/p"));
setupCache.getLoadKeys().add("conan_cache");
return setupCache;
}
private PublishCoverageReportStep newCoverageReportPublishStep() {
var publishCoverageReport = new PublishCoberturaReportStep();
publishCoverageReport.setName("publish coverage report");
publishCoverageReport.setReportName("Coverage");
publishCoverageReport.setFilePatterns("coverage*.xml");
publishCoverageReport.setCondition(ExecuteCondition.ALWAYS);
return publishCoverageReport;
}
private PublishCppcheckReportStep newCppcheckReportPublishStep() {
var publishCppcheckReport = new PublishCppcheckReportStep();
publishCppcheckReport.setName("publish cppcheck report");
publishCppcheckReport.setReportName("Cppcheck");
publishCppcheckReport.setFilePatterns("check-result.xml");
publishCppcheckReport.setCondition(ExecuteCondition.ALWAYS);
return publishCppcheckReport;
}
private Job newJob() {
Job job = new Job();
job.setName("cmake ci");
return job;
}
private void addCommonJobsAndTriggers(Job job) {
var checkout = new CheckoutStep();
checkout.setName("checkout code");
job.getSteps().add(0, checkout);
job.getSteps().add(newCoverageReportPublishStep());
job.getSteps().add(newCppcheckReportPublishStep());
job.getTriggers().add(new BranchUpdateTrigger());
job.getTriggers().add(new PullRequestUpdateTrigger());
}
@Override
public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
List<Job> jobs = new ArrayList<>();
var hasVcpkg = project.getBlob(new BlobIdent(commitId.name(), "vcpkg.json", FileMode.TYPE_FILE), false) != null;
var hasConan = project.getBlob(new BlobIdent(commitId.name(), "conanfile.txt", FileMode.TYPE_FILE), false) != null;
if (project.getBlob(new BlobIdent(commitId.name(), "CMakeLists.txt", FileMode.TYPE_FILE), false) != null) {
Job job = newJob();
if (hasVcpkg) {
job.getSteps().add(newChecksumGenerateStep("vcpkg.json vcpkg-configuration.json"));
job.getSteps().add(newVcpkgCacheSetupStep());
} else if (hasConan) {
job.getSteps().add(newChecksumGenerateStep("conanfile.txt"));
job.getSteps().add(newConanCacheSetupStep());
} else {
job.getSteps().add(newChecksumGenerateStep("**/CMakeLists.txt"));
job.getSteps().add(newFetchContentCacheSetupStep());
}
CommandStep testAndCheck = new CommandStep();
testAndCheck.setName("test and check");
testAndCheck.setImage("1dev/cmake:1.0.0");
var commandsBuilder = new StringBuilder("set -e\n");
if (hasConan)
commandsBuilder.append("conan install . --output-folder=build --build=missing\n");
commandsBuilder.append(
"cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS_RELEASE=\"-g -fprofile-arcs -ftest-coverage\" -DCMAKE_C_FLAGS_RELEASE=\"-g -fprofile-arcs -ftest-coverage\"");
if (hasVcpkg)
commandsBuilder.append(" -DCMAKE_TOOLCHAIN_FILE=\"$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake\"");
else if (hasConan)
commandsBuilder.append(" -DCMAKE_MODULE_PATH=\"$(pwd)/build\"");
commandsBuilder.append(
"\ncmake --build build\n" +
"GTEST_COLOR=1 ctest --test-dir build --verbose\n" +
"gcovr --exclude build --cobertura --cobertura-pretty > coverage.xml\n" +
"cppcheck -ibuild . --xml 2>check-result.xml");
testAndCheck.getInterpreter().setCommands(commandsBuilder.toString());
job.getSteps().add(testAndCheck);
addCommonJobsAndTriggers(job);
jobs.add(job);
}
return jobs;
}
}

20
server-plugin/server-plugin-buildspec-cmake/src/main/java/io/onedev/server/plugin/buildspec/cmake/CmakeModule.java Normal file
View File

@ -0,0 +1,20 @@
package io.onedev.server.plugin.buildspec.cmake;
import io.onedev.commons.loader.AbstractPluginModule;
import io.onedev.server.buildspec.job.JobSuggestion;
/**
* NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
*
*/
public class CmakeModule extends AbstractPluginModule {
@Override
protected void configure() {
super.configure();
// put your guice bindings here
contribute(JobSuggestion.class, CmakeJobSuggestion.class);
}
}

3
server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java
View File

@ -53,7 +53,8 @@ public class DotnetJobSuggestion implements JobSuggestion {
var testAndAnalyze = new CommandStep();
testAndAnalyze.setName("test and analyze");
testAndAnalyze.setImage("mcr.microsoft.com/dotnet/sdk");
testAndAnalyze.getInterpreter().setCommands(
testAndAnalyze.getInterpreter().setCommands("" +
"set -e\n" +
"dotnet tool install -g roslynator.dotnet.cli\n" +
"dotnet test -l trx --collect:\"XPlat Code Coverage\"\n" +
"/root/.dotnet/tools/roslynator analyze -o roslynator-analysis.xml\n");

57
server-plugin/server-plugin-buildspec-python/src/main/java/io/onedev/server/plugin/buildspec/python/PythonJobSuggestion.java
View File

@ -95,7 +95,7 @@ public class PythonJobSuggestion implements JobSuggestion {
testAndLint.setInterpreter(interpreter);
testAndLint.setName("test and lint");
testAndLint.setImage("python");
interpreter.setCommands(dependencyInstallCommand + "\n" + "pip install " + getCoveragePackage(withPytest) + " ruff\n" + getCoverageAndRuffCommand(withPytest, ""));
interpreter.setCommands("set -e\n" + dependencyInstallCommand + "\n" + "pip install " + getCoveragePackage(withPytest) + " ruff\n" + getCoverageAndRuffCommand(withPytest, ""));
return testAndLint;
}
@ -158,17 +158,29 @@ public class PythonJobSuggestion implements JobSuggestion {
return null;
}
private Job newJob() {
var job = new Job();
job.setName("python ci");
return job;
}
private void addCommonStepsAndTriggers(Job job) {
var checkout = new CheckoutStep();
checkout.setName("checkout code");
job.getSteps().add(0, checkout);
job.getSteps().add(newCoverageReportPublishStep());
job.getSteps().add(newRuffReportPublishStep());
job.getTriggers().add(new BranchUpdateTrigger());
job.getTriggers().add(new PullRequestUpdateTrigger());
}
@Override
public Collection<Job> suggestJobs(Project project, ObjectId commitId) {
var jobs = new ArrayList<Job>();
try {
Job job = new Job();
job.setName("python ci");
var checkout = new CheckoutStep();
checkout.setName("checkout code");
job.getSteps().add(checkout);
Blob blob;
if ((blob = project.getBlob(new BlobIdent(commitId.name(), "tox.ini", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
job.getSteps().add(newChecksumGenerateStep("tox.ini"));
var setupCache = new SetupCacheStep();
setupCache.setName("set up dependency cache");
@ -184,7 +196,10 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(testAndLint);
if (blob.getText().getContent().contains("pytest"))
job.getSteps().add(newUnitTestReportPublishStep());
addCommonStepsAndTriggers(job);
jobs.add(job);
} else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "poetry.lock", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
job.getSteps().add(newChecksumGenerateStep("poetry.lock"));
var setupCache = new SetupCacheStep();
@ -222,6 +237,7 @@ public class PythonJobSuggestion implements JobSuggestion {
testAndLint.setName("test and lint");
testAndLint.setImage("1dev/poetry:1.0.1");
String commands = "" +
"set -e\n" +
"poetry config virtualenvs.create false\n" +
installCommand + "\n" +
"poetry add " + getCoveragePackage(withPytest) + " ruff\n";
@ -230,7 +246,10 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(testAndLint);
if (withPytest)
job.getSteps().add(newUnitTestReportPublishStep());
addCommonStepsAndTriggers(job);
jobs.add(job);
} else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "pyproject.toml", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
var dependencyFiles = "pyproject.toml";
var setupPy = project.getBlob(new BlobIdent(commitId.name(), "setup.py", FileMode.TYPE_FILE), false);
if (setupPy != null)
@ -278,7 +297,10 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(newPipTestAndLintStep(installCommand, withPytest));
if (withPytest)
job.getSteps().add(newUnitTestReportPublishStep());
addCommonStepsAndTriggers(job);
jobs.add(job);
} else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "setup.py", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
var dependencyFiles = "setup.py";
var setupCfg = project.getBlob(new BlobIdent(commitId.name(), "setup.cfg", FileMode.TYPE_FILE), false);
if (setupCfg != null)
@ -306,7 +328,10 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(newPipTestAndLintStep(installCommand, withPytest));
if (withPytest)
job.getSteps().add(newUnitTestReportPublishStep());
addCommonStepsAndTriggers(job);
jobs.add(job);
} else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "requirements.txt", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
job.getSteps().add(newChecksumGenerateStep("requirements.txt"));
job.getSteps().add(newPipCacheSetupStep());
@ -314,7 +339,10 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(newPipTestAndLintStep("pip install -r requirements.txt", withPytest));
if (withPytest)
job.getSteps().add(newUnitTestReportPublishStep());
addCommonStepsAndTriggers(job);
jobs.add(job);
} else if ((blob = project.getBlob(new BlobIdent(commitId.name(), "environment.yml", FileMode.TYPE_FILE), false)) != null) {
var job = newJob();
job.getSteps().add(newChecksumGenerateStep("environment.yml"));
var setupCache = new SetupCacheStep();
@ -331,6 +359,7 @@ public class PythonJobSuggestion implements JobSuggestion {
testAndLint.setImage("1dev/conda:1.0.4");
testAndLint.setInterpreter(new ShellInterpreter());
String commands = "" +
"set -e\n" +
"source /root/.bashrc\n" +
"conda env update\n" +
"conda activate " + environments.get("name") + "\n" +
@ -341,21 +370,13 @@ public class PythonJobSuggestion implements JobSuggestion {
job.getSteps().add(testAndLint);
if (withPytest)
job.getSteps().add(newUnitTestReportPublishStep());
}
if (job.getSteps().size() > 1) {
job.getSteps().add(newCoverageReportPublishStep());
job.getSteps().add(newRuffReportPublishStep());
job.getTriggers().add(new BranchUpdateTrigger());
job.getTriggers().add(new PullRequestUpdateTrigger());
return Lists.newArrayList(job);
} else {
return new ArrayList<>();
addCommonStepsAndTriggers(job);
jobs.add(job);
}
} catch (Exception e) {
logger.error("Error suggesting python jobs", e);
return new ArrayList<>();
}
return jobs;
}
}

0
server-plugin/server-plugin-report-cppcheck/src/main/java/io/onedev/server/plugin/report/cppcheck/check-result.xml → server-plugin/server-plugin-report-cppcheck/src/test/java/io/onedev/server/plugin/report/cppcheck/check-result.xml
View File

10
server-product/pom.xml
View File

@ -70,6 +70,16 @@
<artifactId>server-plugin-buildspec-python</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
<artifactId>server-plugin-buildspec-cmake</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
<artifactId>server-plugin-buildspec-bazel</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
<artifactId>server-plugin-buildspec-node</artifactId>