From 9ea3011bbe02cac2e7bb53f7d44c7e346448d69c Mon Sep 17 00:00:00 2001 From: Robin Shen Date: Wed, 16 Nov 2022 15:20:44 +0800 Subject: [PATCH] HA support for markdown reports --- .../buildspec/step/PublishArtifactStep.java | 3 +- .../onedev/server/job/DefaultJobManager.java | 8 +-- .../io/onedev/server/model/PullRequest.java | 2 +- .../detail/dashboard/BuildDashboardPage.java | 32 ++++++++-- .../detail/PullRequestDetailPage.java | 2 +- .../detail/PullRequestSummaryPart.java | 16 +++-- .../report/markdown/MarkdownReportModule.java | 63 +++++++++++++------ .../report/markdown/MarkdownReportPage.java | 42 ++++++++++--- .../report/markdown/MarkdownReportTab.java | 48 +++++++++++--- .../PublishPullRequestMarkdownReportStep.java | 6 +- .../PullRequestSummaryMarkdownPart.java | 43 ++++++++----- 11 files changed, 190 insertions(+), 75 deletions(-) diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java index 9b5908c25b..e639435b24 100644 --- a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java +++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java @@ -73,9 +73,8 @@ public class PublishArtifactStep extends ServerSideStep { @Override public Void call() throws Exception { - File artifactsDir = build.getArtifactsDir(); OneDev.getInstance(StorageManager.class).initArtifactsDir(build.getProject().getId(), build.getNumber()); - FileUtils.copyDirectory(inputDir, artifactsDir); + FileUtils.copyDirectory(inputDir, build.getArtifactsDir()); return null; } diff --git a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java index 4e438a12b5..085aa896bc 100644 --- a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java +++ b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java @@ -1500,11 +1500,11 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz @Override public Void call() throws Exception { - if (dependency.getArtifactsDir().exists()) { + File artifactsDir = dependency.getArtifactsDir(); + if (artifactsDir.exists()) { PatternSet patternSet = PatternSet.parse(dependence.getArtifacts()); - int baseLen = dependency.getArtifactsDir().getAbsolutePath().length()+1; - for (File file: FileUtils.listFiles(dependency.getArtifactsDir(), - patternSet.getIncludes(), patternSet.getExcludes())) { + int baseLen = artifactsDir.getAbsolutePath().length()+1; + for (File file: FileUtils.listFiles(artifactsDir, patternSet.getIncludes(), patternSet.getExcludes())) { FileUtils.copyFile(file, new File(targetDir, file.getAbsolutePath().substring(baseLen))); } diff --git a/server-core/src/main/java/io/onedev/server/model/PullRequest.java b/server-core/src/main/java/io/onedev/server/model/PullRequest.java index 118408d906..7dae31dcea 100644 --- a/server-core/src/main/java/io/onedev/server/model/PullRequest.java +++ b/server-core/src/main/java/io/onedev/server/model/PullRequest.java @@ -811,7 +811,7 @@ public class PullRequest extends AbstractEntity Project project = getTargetProject(); ObjectId headCommitId = ObjectId.fromString(getLatestUpdate().getHeadCommitHash()); ObjectId targetHeadCommitId = ObjectId.fromString(getLatestUpdate().getTargetHeadCommitHash()); - return getGitService().getReachableCommits(project, Lists.newArrayList(headCommitId), + pendingCommits = getGitService().getReachableCommits(project, Lists.newArrayList(headCommitId), Lists.newArrayList(targetHeadCommitId)); } return pendingCommits; diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java index 0c63667bab..48d5677a8c 100644 --- a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java +++ b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java @@ -1,10 +1,16 @@ package io.onedev.server.web.page.project.builds.detail.dashboard; +import java.io.File; + import org.apache.wicket.RestartResponseException; import org.apache.wicket.core.request.handler.PageProvider; import org.apache.wicket.core.request.handler.RenderPageRequestHandler.RedirectPolicy; import org.apache.wicket.request.mapper.parameter.PageParameters; +import io.onedev.server.OneDev; +import io.onedev.server.cluster.ClusterTask; +import io.onedev.server.entitymanager.ProjectManager; +import io.onedev.server.model.Build; import io.onedev.server.security.SecurityUtils; import io.onedev.server.web.page.project.builds.detail.BuildDetailPage; import io.onedev.server.web.page.project.builds.detail.artifacts.BuildArtifactsPage; @@ -19,14 +25,28 @@ public class BuildDashboardPage extends BuildDetailPage { super(params); PageProvider pageProvider; - if (SecurityUtils.canAccessLog(getBuild())) + if (SecurityUtils.canAccessLog(getBuild())) { pageProvider = new PageProvider(BuildLogPage.class, BuildLogPage.paramsOf(getBuild())); - else if (SecurityUtils.canReadCode(getProject())) + } else if (SecurityUtils.canReadCode(getProject())) { pageProvider = new PageProvider(BuildPipelinePage.class, BuildPipelinePage.paramsOf(getBuild())); - else if (getBuild().getArtifactsDir().exists()) - pageProvider = new PageProvider(BuildArtifactsPage.class, BuildArtifactsPage.paramsOf(getBuild())); - else - pageProvider = new PageProvider(FixedIssuesPage.class, FixedIssuesPage.paramsOf(getBuild())); + } else { + Long projectId = getBuild().getProject().getId(); + Long buildNumber = getBuild().getNumber(); + + boolean hasArtifacts = OneDev.getInstance(ProjectManager.class).runOnProjectServer(projectId, new ClusterTask() { + + @Override + public Boolean call() throws Exception { + File artifactsDir = Build.getArtifactsDir(projectId, buildNumber); + return artifactsDir.exists() && artifactsDir.listFiles().length != 0; + } + + }); + if (hasArtifacts) + pageProvider = new PageProvider(BuildArtifactsPage.class, BuildArtifactsPage.paramsOf(getBuild())); + else + pageProvider = new PageProvider(FixedIssuesPage.class, FixedIssuesPage.paramsOf(getBuild())); + } throw new RestartResponseException(pageProvider, RedirectPolicy.NEVER_REDIRECT); } diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestDetailPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestDetailPage.java index fc06dbb0d2..0c109eae13 100644 --- a/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestDetailPage.java +++ b/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestDetailPage.java @@ -1230,7 +1230,7 @@ public abstract class PullRequestDetailPage extends ProjectPage implements PullR @Override protected void populateItem(ListItem item) { PullRequestSummaryPart part = item.getModelObject(); - item.add(new Label("head", part.getTitle())); + item.add(new Label("head", part.getReportName())); item.add(part.render("body")); } diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestSummaryPart.java b/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestSummaryPart.java index f2ff67ed82..308da46f80 100644 --- a/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestSummaryPart.java +++ b/server-core/src/main/java/io/onedev/server/web/page/project/pullrequests/detail/PullRequestSummaryPart.java @@ -1,17 +1,21 @@ package io.onedev.server.web.page.project.pullrequests.detail; +import java.io.Serializable; + import org.apache.wicket.Component; -public abstract class PullRequestSummaryPart { +public abstract class PullRequestSummaryPart implements Serializable { - private final String title; + private static final long serialVersionUID = 1L; - public PullRequestSummaryPart(String title) { - this.title = title; + private final String reportName; + + public PullRequestSummaryPart(String reportName) { + this.reportName = reportName; } - public String getTitle() { - return title; + public String getReportName() { + return reportName; } public abstract Component render(String componentId); diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java index 4dcd0e8bce..1bd59707a7 100644 --- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java +++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java @@ -58,11 +58,10 @@ public class MarkdownReportModule extends AbstractPluginModule { @Override public List getTabs(Build build) { - ProjectManager projectManager = OneDev.getInstance(ProjectManager.class); Long projectId = build.getProject().getId(); Long buildNumber = build.getNumber(); - return projectManager.runOnProjectServer(projectId, new GetBuildTabs(projectId, buildNumber)).stream() + return getProjectManager().runOnProjectServer(projectId, new GetBuildTabs(projectId, buildNumber)).stream() .filter(it->SecurityUtils.canAccessReport(build, it.getTitle())) .collect(Collectors.toList()); } @@ -79,23 +78,13 @@ public class MarkdownReportModule extends AbstractPluginModule { @Override public List getParts(PullRequest request) { List parts = new ArrayList<>(); + Long projectId = request.getProject().getId(); for (Build build: request.getCurrentBuilds()) { - parts.addAll(LockUtils.read(PublishPullRequestMarkdownReportStep.getReportLockKey(build), new Callable>() { - - @Override - public List call() throws Exception { - List parts = new ArrayList<>(); - File categoryDir = new File(build.getDir(), PublishPullRequestMarkdownReportStep.CATEGORY); - if (categoryDir.exists()) { - for (File reportDir: categoryDir.listFiles()) { - if (SecurityUtils.canAccessReport(build, reportDir.getName())) - parts.add(new PullRequestSummaryMarkdownPart(reportDir)); - } - } - return parts; - } - - })); + Long buildNumber = build.getNumber(); + for (PullRequestSummaryPart part: getProjectManager().runOnProjectServer(projectId, new GetPullRequestSummaryParts(projectId, buildNumber))) { + if (SecurityUtils.canAccessReport(build, part.getReportName())) + parts.add(part); + } } return parts; } @@ -119,6 +108,10 @@ public class MarkdownReportModule extends AbstractPluginModule { }); } + + private ProjectManager getProjectManager() { + return OneDev.getInstance(ProjectManager.class); + } private static class GetBuildTabs implements ClusterTask> { @@ -140,7 +133,7 @@ public class MarkdownReportModule extends AbstractPluginModule { @Override public List call() throws Exception { List tabs = new ArrayList<>(); - File categoryDir = new File(Build.getDir(projectId, buildNumber), PublishPullRequestMarkdownReportStep.CATEGORY); + File categoryDir = new File(Build.getDir(projectId, buildNumber), PublishMarkdownReportStep.CATEGORY); if (categoryDir.exists()) { for (File reportDir: categoryDir.listFiles()) tabs.add(new MarkdownReportTab(reportDir.getName())); @@ -161,4 +154,36 @@ public class MarkdownReportModule extends AbstractPluginModule { } + private static class GetPullRequestSummaryParts implements ClusterTask> { + + private static final long serialVersionUID = 1L; + + private final Long projectId; + + private final Long buildNumber; + + private GetPullRequestSummaryParts(Long projectId, Long buildNumber) { + this.projectId = projectId; + this.buildNumber = buildNumber; + } + + @Override + public List call() throws Exception { + return LockUtils.read(PublishPullRequestMarkdownReportStep.getReportLockName(projectId, buildNumber), new Callable>() { + + @Override + public List call() throws Exception { + List parts = new ArrayList<>(); + File categoryDir = new File(Build.getDir(projectId, buildNumber), PublishPullRequestMarkdownReportStep.CATEGORY); + if (categoryDir.exists()) { + for (File reportDir: categoryDir.listFiles()) + parts.add(new PullRequestSummaryMarkdownPart(projectId, buildNumber, reportDir.getName())); + } + return parts; + } + + }); + } + + } } diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java index 311526d16d..8debaedb5e 100644 --- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java +++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java @@ -1,7 +1,6 @@ package io.onedev.server.plugin.report.markdown; import java.io.File; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -19,6 +18,9 @@ import com.google.common.base.Splitter; import io.onedev.commons.utils.FileUtils; import io.onedev.commons.utils.StringUtils; +import io.onedev.server.OneDev; +import io.onedev.server.cluster.ClusterTask; +import io.onedev.server.entitymanager.ProjectManager; import io.onedev.server.model.Build; import io.onedev.server.security.SecurityUtils; import io.onedev.server.web.component.markdown.MarkdownViewer; @@ -57,14 +59,10 @@ public class MarkdownReportPage extends BuildDetailPage { protected void onInitialize() { super.onInitialize(); - File file = new File(getBuild().getDir(), - PublishMarkdownReportStep.CATEGORY + "/" + reportName + "/" + filePath); - try { - String markdown = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - add(new MarkdownViewer("markdownReport", Model.of(markdown), null)); - } catch (IOException e) { - throw new RuntimeException(e); - } + Long projectId = getBuild().getProject().getId(); + ProjectManager projectManager = OneDev.getInstance(ProjectManager.class); + String markdown = projectManager.runOnProjectServer(projectId, new GetMarkdownContent(projectId, getBuild().getNumber(), reportName, filePath)); + add(new MarkdownViewer("markdownReport", Model.of(markdown), null)); } @Override @@ -94,4 +92,30 @@ public class MarkdownReportPage extends BuildDetailPage { } return params; } + + private static class GetMarkdownContent implements ClusterTask { + + private final Long projectId; + + private final Long buildNumber; + + private final String reportName; + + private final String filePath; + + private GetMarkdownContent(Long projectId, Long buildNumber, String reportName, String filePath) { + this.projectId = projectId; + this.buildNumber = buildNumber; + this.reportName = reportName; + this.filePath = filePath; + } + + @Override + public String call() throws Exception { + File file = new File(Build.getDir(projectId, buildNumber), + PublishMarkdownReportStep.CATEGORY + "/" + reportName + "/" + filePath); + return FileUtils.readFileToString(file, StandardCharsets.UTF_8); + } + + } } diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java index b1255b26cb..c37e00560a 100644 --- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java +++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java @@ -11,6 +11,10 @@ import org.apache.wicket.request.mapper.parameter.PageParameters; import io.onedev.commons.utils.FileUtils; import io.onedev.commons.utils.LockUtils; +import io.onedev.server.OneDev; +import io.onedev.server.cluster.ClusterTask; +import io.onedev.server.entitymanager.ProjectManager; +import io.onedev.server.model.Build; import io.onedev.server.web.component.link.ViewStateAwarePageLink; import io.onedev.server.web.component.tabbable.PageTabHead; import io.onedev.server.web.page.project.builds.detail.BuildDetailPage; @@ -30,16 +34,11 @@ public class MarkdownReportTab extends BuildTab { @Override protected Link newLink(String linkId, Class pageClass) { BuildDetailPage page = (BuildDetailPage) getPage(); - String startPage = LockUtils.read(PublishMarkdownReportStep.getReportLockName(page.getBuild()), new Callable() { - - @Override - public String call() throws Exception { - File startPageFile = new File(page.getBuild().getDir(), - PublishMarkdownReportStep.CATEGORY + "/" + getTitle() + "/" + PublishMarkdownReportStep.START_PAGE); - return FileUtils.readFileToString(startPageFile, StandardCharsets.UTF_8); - } - - }); + Build build = page.getBuild(); + Long projectId = build.getProject().getId(); + Long buildNumber = build.getNumber(); + ProjectManager projectManager = OneDev.getInstance(ProjectManager.class); + String startPage = projectManager.runOnProjectServer(projectId, new GetStartPage(projectId, buildNumber, getTitle())); PageParameters params = MarkdownReportPage.paramsOf(page.getBuild(), getTitle(), startPage); return new ViewStateAwarePageLink(linkId, pageClass, params); @@ -58,4 +57,33 @@ public class MarkdownReportTab extends BuildTab { } } + private static class GetStartPage implements ClusterTask { + + private Long projectId; + + private Long buildNumber; + + private String reportName; + + private GetStartPage(Long projectId, Long buildNumber, String reportName) { + this.projectId = projectId; + this.buildNumber = buildNumber; + this.reportName = reportName; + } + + @Override + public String call() throws Exception { + return LockUtils.read(PublishMarkdownReportStep.getReportLockName(projectId, buildNumber), new Callable() { + + @Override + public String call() throws Exception { + File startPageFile = new File(Build.getDir(projectId, buildNumber), + PublishMarkdownReportStep.CATEGORY + "/" + reportName + "/" + PublishMarkdownReportStep.START_PAGE); + return FileUtils.readFileToString(startPageFile, StandardCharsets.UTF_8); + } + + }); + } + + } } diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java index 6d40edda4c..c884fb2bef 100644 --- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java +++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java @@ -55,7 +55,7 @@ public class PublishPullRequestMarkdownReportStep extends PublishReportStep { @Override public Map run(Build build, File workspace, TaskLogger logger) { if (build.getRequest() != null) { - LockUtils.write(getReportLockKey(build), new Callable() { + LockUtils.write(getReportLockName(build.getProject().getId(), build.getNumber()), new Callable() { @Override public Void call() throws Exception { @@ -76,8 +76,8 @@ public class PublishPullRequestMarkdownReportStep extends PublishReportStep { return null; } - public static String getReportLockKey(Build build) { - return PublishPullRequestMarkdownReportStep.class.getName() + ":" + build.getId(); + public static String getReportLockName(Long projectId, Long buildNumber) { + return PublishPullRequestMarkdownReportStep.class.getName() + ":" + projectId + ":" + buildNumber; } } diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java index 22caa1322b..2ba68c340e 100644 --- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java +++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java @@ -1,7 +1,6 @@ package io.onedev.server.plugin.report.markdown; import java.io.File; -import java.io.IOException; import java.nio.charset.StandardCharsets; import org.apache.wicket.Component; @@ -9,28 +8,44 @@ import org.apache.wicket.behavior.AttributeAppender; import org.apache.wicket.model.Model; import io.onedev.commons.utils.FileUtils; +import io.onedev.server.OneDev; +import io.onedev.server.cluster.ClusterTask; +import io.onedev.server.entitymanager.ProjectManager; +import io.onedev.server.model.Build; import io.onedev.server.web.component.markdown.MarkdownViewer; import io.onedev.server.web.page.project.pullrequests.detail.PullRequestSummaryPart; public class PullRequestSummaryMarkdownPart extends PullRequestSummaryPart { - private final File reportDir; + private static final long serialVersionUID = 1L; - public PullRequestSummaryMarkdownPart(File reportDir) { - super(reportDir.getName()); - this.reportDir = reportDir; + private final Long projectId; + + private final Long buildNumber; + + public PullRequestSummaryMarkdownPart(Long projectId, Long buildNumber, String reportName) { + super(reportName); + this.projectId = projectId; + this.buildNumber = buildNumber; } - + @Override public Component render(String componentId) { - try { - File file = new File(reportDir, PublishPullRequestMarkdownReportStep.FILE); - String markdown = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - return new MarkdownViewer(componentId, Model.of(markdown), null) - .add(AttributeAppender.append("class", "mb-n3")); - } catch (IOException e) { - throw new RuntimeException(e); - } + ProjectManager projectManager = OneDev.getInstance(ProjectManager.class); + String markdown = projectManager.runOnProjectServer(projectId, new ClusterTask() { + + private static final long serialVersionUID = 1L; + + @Override + public String call() throws Exception { + File categoryDir = new File(Build.getDir(projectId, buildNumber), PublishPullRequestMarkdownReportStep.CATEGORY); + File file = new File(new File(categoryDir, getReportName()), PublishPullRequestMarkdownReportStep.FILE); + return FileUtils.readFileToString(file, StandardCharsets.UTF_8); + } + + }); + return new MarkdownViewer(componentId, Model.of(markdown), null) + .add(AttributeAppender.append("class", "mb-n3")); } }