archive-gh-me/gitpod
1
0
Fork 0
mirror of https://github.com/gitpod-io/gitpod.git synced 2025-12-08 17:36:30 +00:00
Code Issues Projects Releases Wiki Activity

[multi-repo] Support prebuilds on multi-repo setup

Browse Source
This commit is contained in:
Sven Efftinge 2022-03-09 17:42:50 +00:00 committed by Robo Quat
parent 2e1774eaf7
commit cc2f3b30fe
70 changed files with 943 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
components/content-service-api/go/blobs_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/content_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/headless-log_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/ideplugin_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

110
components/content-service-api/go/initializer.pb.go
View File

@ -662,9 +662,8 @@ type PrebuildInitializer struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Prebuild *SnapshotInitializer `protobuf:"bytes,1,opt,name=prebuild,proto3" json:"prebuild,omitempty"`
Git *GitInitializer `protobuf:"bytes,2,opt,name=git,proto3" json:"git,omitempty"`
Composite *CompositeInitializer `protobuf:"bytes,3,opt,name=composite,proto3" json:"composite,omitempty"`
Prebuild *SnapshotInitializer `protobuf:"bytes,1,opt,name=prebuild,proto3" json:"prebuild,omitempty"`
Git []*GitInitializer `protobuf:"bytes,2,rep,name=git,proto3" json:"git,omitempty"`
}
func (x *PrebuildInitializer) Reset() {
@ -706,20 +705,13 @@ func (x *PrebuildInitializer) GetPrebuild() *SnapshotInitializer {
return nil
}
func (x *PrebuildInitializer) GetGit() *GitInitializer {
func (x *PrebuildInitializer) GetGit() []*GitInitializer {
if x != nil {
return x.Git
}
return nil
}
func (x *PrebuildInitializer) GetComposite() *CompositeInitializer {
if x != nil {
return x.Composite
}
return nil
}
// FromBackupInitializer initializes content from a previously made backup
type FromBackupInitializer struct {
state protoimpl.MessageState
@ -1039,58 +1031,53 @@ var file_initializer_proto_rawDesc = []byte{
0x3a, 0x02, 0x38, 0x01, 0x22, 0x31, 0x0a, 0x13, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x22, 0xcc, 0x01, 0x0a, 0x13, 0x50, 0x72, 0x65, 0x62,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x22, 0x88, 0x01, 0x0a, 0x13, 0x50, 0x72, 0x65, 0x62,
0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12,
0x3f, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69,
0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64,
0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e,
0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e,
0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47,
0x69, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x03, 0x67,
0x69, 0x74, 0x12, 0x42, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65,
0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6d,
0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61,
0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x22,
0xe7, 0x02, 0x0a, 0x09, 0x47, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a,
0x06, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62,
0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f,
0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x61,
0x74, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e,
0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64,
0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75,
0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18,
0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x6f,
0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x75,
0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46,
0x69, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e,
0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20,
0x01, 0x28, 0x03, 0x52, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x74, 0x72, 0x61, 0x63,
0x6b, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x70, 0x75,
0x73, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03,
0x28, 0x09, 0x52, 0x0f, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d,
0x69, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x70,
0x75, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x08, 0x20,
0x01, 0x28, 0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x70, 0x75, 0x73, 0x68,
0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x2a, 0x5a, 0x0a, 0x0f, 0x43, 0x6c, 0x6f,
0x6e, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b,
0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a,
0x0d, 0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01,
0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43,
0x48, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x42, 0x52, 0x41,
0x4e, 0x43, 0x48, 0x10, 0x03, 0x2a, 0x40, 0x0a, 0x0d, 0x47, 0x69, 0x74, 0x41, 0x75, 0x74, 0x68,
0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x5f, 0x41, 0x55, 0x54,
0x48, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54,
0x48, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54,
0x48, 0x5f, 0x4f, 0x54, 0x53, 0x10, 0x02, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2d, 0x69, 0x6f, 0x2f,
0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x69, 0x74, 0x22, 0x17, 0x0a, 0x15, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70,
0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x22, 0xe7, 0x02, 0x0a, 0x09,
0x47, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x72, 0x61,
0x6e, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x72, 0x61, 0x6e, 0x63,
0x68, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d,
0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74,
0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d,
0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
0x52, 0x0f, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65,
0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x63, 0x6f, 0x6d,
0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x75, 0x6e, 0x74, 0x72, 0x61,
0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09,
0x52, 0x0e, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73,
0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63,
0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52,
0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46,
0x69, 0x6c, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64,
0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f,
0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12,
0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65,
0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52,
0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f,
0x6d, 0x6d, 0x69, 0x74, 0x73, 0x2a, 0x5a, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x61,
0x72, 0x67, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x4d, 0x4f,
0x54, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x4d,
0x4f, 0x54, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d,
0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43, 0x48, 0x10, 0x02, 0x12,
0x10, 0x0a, 0x0c, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43, 0x48, 0x10,
0x03, 0x2a, 0x40, 0x0a, 0x0d, 0x47, 0x69, 0x74, 0x41, 0x75, 0x74, 0x68, 0x4d, 0x65, 0x74, 0x68,
0x6f, 0x64, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x10, 0x00, 0x12,
0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x10, 0x01, 0x12,
0x12, 0x0a, 0x0e, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x5f, 0x4f, 0x54,
0x53, 0x10, 0x02, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2d, 0x69, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x70,
0x6f, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -1139,12 +1126,11 @@ var file_initializer_proto_depIdxs = []int32{
1, // 12: contentservice.GitConfig.authentication:type_name -> contentservice.GitAuthMethod
8, // 13: contentservice.PrebuildInitializer.prebuild:type_name -> contentservice.SnapshotInitializer
6, // 14: contentservice.PrebuildInitializer.git:type_name -> contentservice.GitInitializer
3, // 15: contentservice.PrebuildInitializer.composite:type_name -> contentservice.CompositeInitializer
16, // [16:16] is the sub-list for method output_type
16, // [16:16] is the sub-list for method input_type
16, // [16:16] is the sub-list for extension type_name
16, // [16:16] is the sub-list for extension extendee
0, // [0:16] is the sub-list for field type_name
15, // [15:15] is the sub-list for method output_type
15, // [15:15] is the sub-list for method input_type
15, // [15:15] is the sub-list for extension type_name
15, // [15:15] is the sub-list for extension extendee
0, // [0:15] is the sub-list for field type_name
}
func init() { file_initializer_proto_init() }

2
components/content-service-api/go/workspace_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

3
components/content-service-api/initializer.proto
View File

@ -118,8 +118,7 @@ message SnapshotInitializer {
// If restoring the snapshot fails, we fall back to a regular Git initializer, which might be composite git initializer for multi-repo projects.
message PrebuildInitializer {
SnapshotInitializer prebuild = 1;
GitInitializer git = 2;
CompositeInitializer composite = 3;
repeated GitInitializer git = 2;
}
// FromBackupInitializer initializes content from a previously made backup

2
components/content-service-api/typescript/src/blobs_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/blobs_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/blobs_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/content_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/content_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/content_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/headless-log_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/headless-log_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/ideplugin_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/ideplugin_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/initializer_grpc_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

17
components/content-service-api/typescript/src/initializer_pb.d.ts vendored
View File

@ -275,16 +275,10 @@ export class PrebuildInitializer extends jspb.Message {
clearPrebuild(): void;
getPrebuild(): SnapshotInitializer | undefined;
setPrebuild(value?: SnapshotInitializer): PrebuildInitializer;
hasGit(): boolean;
clearGit(): void;
getGit(): GitInitializer | undefined;
setGit(value?: GitInitializer): PrebuildInitializer;
hasComposite(): boolean;
clearComposite(): void;
getComposite(): CompositeInitializer | undefined;
setComposite(value?: CompositeInitializer): PrebuildInitializer;
clearGitList(): void;
getGitList(): Array<GitInitializer>;
setGitList(value: Array<GitInitializer>): PrebuildInitializer;
addGit(value?: GitInitializer, index?: number): GitInitializer;
serializeBinary(): Uint8Array;
toObject(includeInstance?: boolean): PrebuildInitializer.AsObject;
@ -299,8 +293,7 @@ export class PrebuildInitializer extends jspb.Message {
export namespace PrebuildInitializer {
export type AsObject = {
prebuild?: SnapshotInitializer.AsObject,
git?: GitInitializer.AsObject,
composite?: CompositeInitializer.AsObject,
gitList: Array<GitInitializer.AsObject>,
}
}

112
components/content-service-api/typescript/src/initializer_pb.js
View File

@ -220,7 +220,7 @@ if (goog.DEBUG && !COMPILED) {
* @constructor
*/
proto.contentservice.PrebuildInitializer = function(opt_data) {
jspb.Message.initialize(this, opt_data, 0, -1, null, null);
jspb.Message.initialize(this, opt_data, 0, -1, proto.contentservice.PrebuildInitializer.repeatedFields_, null);
};
goog.inherits(proto.contentservice.PrebuildInitializer, jspb.Message);
if (goog.DEBUG && !COMPILED) {
@ -2086,6 +2086,13 @@ proto.contentservice.SnapshotInitializer.prototype.setSnapshot = function(value)
/**
* List of repeated fields within this message type.
* @private {!Array<number>}
* @const
*/
proto.contentservice.PrebuildInitializer.repeatedFields_ = [2];
if (jspb.Message.GENERATE_TO_OBJECT) {
@ -2118,8 +2125,8 @@ proto.contentservice.PrebuildInitializer.prototype.toObject = function(opt_inclu
proto.contentservice.PrebuildInitializer.toObject = function(includeInstance, msg) {
var f, obj = {
prebuild: (f = msg.getPrebuild()) && proto.contentservice.SnapshotInitializer.toObject(includeInstance, f),
git: (f = msg.getGit()) && proto.contentservice.GitInitializer.toObject(includeInstance, f),
composite: (f = msg.getComposite()) && proto.contentservice.CompositeInitializer.toObject(includeInstance, f)
gitList: jspb.Message.toObjectList(msg.getGitList(),
proto.contentservice.GitInitializer.toObject, includeInstance)
};
if (includeInstance) {
@ -2164,12 +2171,7 @@ proto.contentservice.PrebuildInitializer.deserializeBinaryFromReader = function(
case 2:
var value = new proto.contentservice.GitInitializer;
reader.readMessage(value,proto.contentservice.GitInitializer.deserializeBinaryFromReader);
msg.setGit(value);
break;
case 3:
var value = new proto.contentservice.CompositeInitializer;
reader.readMessage(value,proto.contentservice.CompositeInitializer.deserializeBinaryFromReader);
msg.setComposite(value);
msg.addGit(value);
break;
default:
reader.skipField();
@ -2208,22 +2210,14 @@ proto.contentservice.PrebuildInitializer.serializeBinaryToWriter = function(mess
proto.contentservice.SnapshotInitializer.serializeBinaryToWriter
);
}
f = message.getGit();
if (f != null) {
writer.writeMessage(
f = message.getGitList();
if (f.length > 0) {
writer.writeRepeatedMessage(
2,
f,
proto.contentservice.GitInitializer.serializeBinaryToWriter
);
}
f = message.getComposite();
if (f != null) {
writer.writeMessage(
3,
f,
proto.contentservice.CompositeInitializer.serializeBinaryToWriter
);
}
};
@ -2265,76 +2259,40 @@ proto.contentservice.PrebuildInitializer.prototype.hasPrebuild = function() {
/**
* optional GitInitializer git = 2;
* @return {?proto.contentservice.GitInitializer}
* repeated GitInitializer git = 2;
* @return {!Array<!proto.contentservice.GitInitializer>}
*/
proto.contentservice.PrebuildInitializer.prototype.getGit = function() {
return /** @type{?proto.contentservice.GitInitializer} */ (
jspb.Message.getWrapperField(this, proto.contentservice.GitInitializer, 2));
proto.contentservice.PrebuildInitializer.prototype.getGitList = function() {
return /** @type{!Array<!proto.contentservice.GitInitializer>} */ (
jspb.Message.getRepeatedWrapperField(this, proto.contentservice.GitInitializer, 2));
};
/**
* @param {?proto.contentservice.GitInitializer|undefined} value
* @param {!Array<!proto.contentservice.GitInitializer>} value
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.setGit = function(value) {
return jspb.Message.setWrapperField(this, 2, value);
proto.contentservice.PrebuildInitializer.prototype.setGitList = function(value) {
return jspb.Message.setRepeatedWrapperField(this, 2, value);
};
/**
* Clears the message field making it undefined.
* @param {!proto.contentservice.GitInitializer=} opt_value
* @param {number=} opt_index
* @return {!proto.contentservice.GitInitializer}
*/
proto.contentservice.PrebuildInitializer.prototype.addGit = function(opt_value, opt_index) {
return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.contentservice.GitInitializer, opt_index);
};
/**
* Clears the list making it empty but non-null.
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.clearGit = function() {
return this.setGit(undefined);
};
/**
* Returns whether this field is set.
* @return {boolean}
*/
proto.contentservice.PrebuildInitializer.prototype.hasGit = function() {
return jspb.Message.getField(this, 2) != null;
};
/**
* optional CompositeInitializer composite = 3;
* @return {?proto.contentservice.CompositeInitializer}
*/
proto.contentservice.PrebuildInitializer.prototype.getComposite = function() {
return /** @type{?proto.contentservice.CompositeInitializer} */ (
jspb.Message.getWrapperField(this, proto.contentservice.CompositeInitializer, 3));
};
/**
* @param {?proto.contentservice.CompositeInitializer|undefined} value
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.setComposite = function(value) {
return jspb.Message.setWrapperField(this, 3, value);
};
/**
* Clears the message field making it undefined.
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.clearComposite = function() {
return this.setComposite(undefined);
};
/**
* Returns whether this field is set.
* @return {boolean}
*/
proto.contentservice.PrebuildInitializer.prototype.hasComposite = function() {
return jspb.Message.getField(this, 3) != null;
proto.contentservice.PrebuildInitializer.prototype.clearGitList = function() {
return this.setGitList([]);
};

2
components/content-service-api/typescript/src/workspace_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/workspace_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

2
components/content-service-api/typescript/src/workspace_pb.js
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

3
components/content-service/pkg/initializer/git.go
View File

@ -141,7 +141,8 @@ func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) {
if ws.TargetMode == RemoteBranch {
// create local branch based on specific remote branch
if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/"+ws.CloneTarget); err != nil {
return err
log.WithField("remoteURI", ws.RemoteURI).WithField("branch", ws.CloneTarget).Debug("Remote branch doesn't exist.")
return nil
}
} else if ws.TargetMode == LocalBranch {
// checkout local branch based on remote HEAD

19
components/content-service/pkg/initializer/initializer.go
View File

@ -111,14 +111,6 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader
if ir.Prebuild == nil {
return nil, status.Error(codes.InvalidArgument, "missing prebuild initializer spec")
}
if ir.Prebuild.Git == nil {
return nil, status.Error(codes.InvalidArgument, "missing prebuild Git initializer spec")
}
gitinit, err := newGitInitializer(ctx, loc, ir.Prebuild.Git, opts.ForceGitpodUserForGit)
if err != nil {
return nil, err
}
var snapshot *SnapshotInitializer
if ir.Prebuild.Prebuild != nil {
snapshot, err = newSnapshotInitializer(loc, rs, ir.Prebuild.Prebuild)
@ -126,10 +118,17 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader
return nil, status.Error(codes.Internal, fmt.Sprintf("cannot setup prebuild init: %v", err))
}
}
var gits []*GitInitializer
for _, gi := range ir.Prebuild.Git {
gitinit, err := newGitInitializer(ctx, loc, gi, opts.ForceGitpodUserForGit)
if err != nil {
return nil, err
}
gits = append(gits, gitinit)
}
initializer = &PrebuildInitializer{
Git: gitinit,
Prebuild: snapshot,
Git: gits,
}
} else if ir, ok := spec.(*csapi.WorkspaceInitializer_Snapshot); ok {
initializer, err = newSnapshotInitializer(loc, rs, ir.Snapshot)

100
components/content-service/pkg/initializer/prebuild.go
View File

@ -26,7 +26,7 @@ import (
// PrebuildInitializer first tries to restore the snapshot/prebuild and if that succeeds performs Git operations.
// If restoring the prebuild does not succeed we fall back to Git entriely.
type PrebuildInitializer struct {
Git *GitInitializer
Git []*GitInitializer
Prebuild *SnapshotInitializer
}
@ -45,14 +45,11 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
tracelog.String("snapshot", p.Prebuild.Snapshot),
)
}
if p.Git == nil {
if len(p.Git) == 0 {
spandata = append(spandata, tracelog.Bool("hasGit", false))
} else {
spandata = append(spandata,
tracelog.Bool("hasGit", true),
tracelog.String("git.targetMode", string(p.Git.TargetMode)),
tracelog.String("git.cloneTarget", string(p.Git.CloneTarget)),
tracelog.String("git.location", string(p.Git.Location)),
)
}
span.LogFields(spandata...)
@ -71,7 +68,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: %w", err)
}
return p.Git.Run(ctx, mappings)
for _, gi := range p.Git {
_, err = gi.Run(ctx, mappings)
if err != nil {
return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: Git fallback: %w", err)
}
}
}
}
@ -80,47 +82,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
src = csapi.WorkspaceInitFromPrebuild
// make sure we're on the correct branch
span.LogFields(
tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(p.Git.Location))),
tracelog.String("location", fmt.Sprintf("%v", p.Git.Location)),
)
if git.IsWorkingCopy(p.Git.Location) {
out, err := p.Git.GitWithOutput(ctx, "stash", "push", "-u")
for _, gi := range p.Git {
err = runGitInit(ctx, gi)
if err != nil {
var giterr git.OpFailedError
if errors.As(err, &giterr) && strings.Contains(giterr.Output, "You do not have the initial commit yet") {
// git stash push returns a non-zero exit code if the repository does not have a single commit.
// In this case that's not an error though, hence we don't want to fail here.
} else {
// git returned a non-zero exit code because of some reason we did not anticipate or an actual failure.
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
return src, err
}
didStash := !strings.Contains(string(out), "No local changes to save")
err = p.Git.Fetch(ctx)
if err != nil {
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
err = p.Git.realizeCloneTarget(ctx)
if err != nil {
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
// If any of these cleanup operations fail that's no reason to fail ws initialization.
// It just results in a slightly degraded state.
if didStash {
err = p.Git.Git(ctx, "stash", "pop")
if err != nil {
// If restoring the stashed changes produces merge conflicts on the new Git ref, simply
// throw them away (they'll remain in the stash, but are likely outdated anyway).
_ = p.Git.Git(ctx, "reset", "--hard")
}
}
log.Debug("prebuild initializer Git operations complete")
}
log.Debug("Initialized workspace with prebuilt snapshot")
return
}
@ -138,3 +105,48 @@ func clearWorkspace(location string) error {
}
return nil
}
func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) {
span, ctx := opentracing.StartSpanFromContext(ctx, "runGitInit")
span.LogFields(
tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(gInit.Location))),
tracelog.String("location", fmt.Sprintf("%v", gInit.Location)),
)
if git.IsWorkingCopy(gInit.Location) {
out, err := gInit.GitWithOutput(ctx, "stash", "push", "-u")
if err != nil {
var giterr git.OpFailedError
if errors.As(err, &giterr) && strings.Contains(giterr.Output, "You do not have the initial commit yet") {
// git stash push returns a non-zero exit code if the repository does not have a single commit.
// In this case that's not an error though, hence we don't want to fail here.
} else {
// git returned a non-zero exit code because of some reason we did not anticipate or an actual failure.
return xerrors.Errorf("prebuild initializer: %w", err)
}
}
didStash := !strings.Contains(string(out), "No local changes to save")
err = gInit.Fetch(ctx)
if err != nil {
return xerrors.Errorf("prebuild initializer: %w", err)
}
err = gInit.realizeCloneTarget(ctx)
if err != nil {
return xerrors.Errorf("prebuild initializer: %w", err)
}
// If any of these cleanup operations fail that's no reason to fail ws initialization.
// It just results in a slightly degraded state.
if didStash {
err = gInit.Git(ctx, "stash", "pop")
if err != nil {
// If restoring the stashed changes produces merge conflicts on the new Git ref, simply
// throw them away (they'll remain in the stash, but are likely outdated anyway).
_ = gInit.Git(ctx, "reset", "--hard")
}
}
log.Debug("prebuild initializer Git operations complete")
}
return nil
}

16
components/content-service/pkg/layer/provider.go
View File

@ -184,7 +184,21 @@ func (s *Provider) GetContentLayer(ctx context.Context, owner, workspaceID strin
span.LogKV("fallback-to-git", err.Error())
// we failed creating a prebuild initializer, so let's try falling back to the Git part.
initializer = &csapi.WorkspaceInitializer{Spec: &csapi.WorkspaceInitializer_Git{Git: pis.Git}}
var init []*csapi.WorkspaceInitializer
for _, gi := range pis.Git {
init = append(init, &csapi.WorkspaceInitializer{
Spec: &csapi.WorkspaceInitializer_Git{
Git: gi,
},
})
}
initializer = &csapi.WorkspaceInitializer{
Spec: &csapi.WorkspaceInitializer_Composite{
Composite: &csapi.CompositeInitializer{
Initializer: init,
},
},
}
} else {
// creating the initializer worked - we're done here
return

12
components/dashboard/src/projects/Prebuilds.tsx
View File

@ -219,14 +219,18 @@ export default function (props: { project?: Project, isAdminDashboard?: boolean
</ItemField>
<ItemField className="flex items-center my-auto">
<div className="truncate">
<div className="text-base text-gray-500 dark:text-gray-50 font-medium mb-1 truncate" title={shortCommitMessage(p.info.changeTitle)}>{shortCommitMessage(p.info.changeTitle)}</div>
<a href={p.info.changeUrl} className="cursor-pointer">
<div className="text-base text-gray-500 dark:text-gray-50 font-medium mb-1 truncate" title={shortCommitMessage(p.info.changeTitle)}>{shortCommitMessage(p.info.changeTitle)}</div>
</a>
<p>{p.info.changeAuthorAvatar && <img className="rounded-full w-4 h-4 inline-block align-text-bottom mr-2" src={p.info.changeAuthorAvatar || ''} alt={p.info.changeAuthor} />}Authored {formatDate(p.info.changeDate)} · {p.info.changeHash?.substring(0, 8)}</p>
</div>
</ItemField>
<ItemField className="flex">
<div className="flex space-x-2 truncate">
<span className="font-medium text-gray-500 dark:text-gray-50 truncate" title={p.info.branch}>{p.info.branch}</span>
</div>
<a href={p.info.changeUrl} className="cursor-pointer">
<div className="flex space-x-2 truncate">
<span className="font-medium text-gray-500 dark:text-gray-50 truncate" title={p.info.branch}>{p.info.branch}</span>
</div>
</a>
<span className="flex-grow" />
{!props.isAdminDashboard && <ItemFieldContextMenu menuEntries={prebuildContextMenu(p)} />}
</ItemField>

6
components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts
View File

@ -31,6 +31,12 @@ export class DBPrebuiltWorkspaceUpdatable implements PrebuiltWorkspaceUpdatable
@Column()
repo: string;
@Column({
default: '',
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED
})
commitSHA?: string;
@Column()
isResolved: boolean;

21
components/gitpod-db/src/typeorm/migration/1646803519382-PrebuildUpdatableSHA.ts Normal file
View File

@ -0,0 +1,21 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
import {MigrationInterface, QueryRunner} from "typeorm";
import { columnExists } from "./helper/helper";
export class PrebuildUpdatableSHA1646803519382 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
if (!(await columnExists(queryRunner, "d_b_prebuilt_workspace_updatable", "commitSHA"))) {
await queryRunner.query("ALTER TABLE d_b_prebuilt_workspace_updatable ADD COLUMN commitSHA varchar(255) NOT NULL");
}
}
public async down(queryRunner: QueryRunner): Promise<void> {
}
}

3
components/gitpod-db/src/typeorm/workspace-db-impl.ts
View File

@ -633,8 +633,9 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
let query = repo.createQueryBuilder('pws');
query = query.where('pws.cloneURL = :cloneURL', { cloneURL })
query = query.orderBy('pws.creationTime', 'ASC');
query = query.orderBy('pws.creationTime', 'DESC');
query = query.innerJoinAndMapOne('pws.workspace', DBWorkspace, 'ws', 'pws.buildWorkspaceId = ws.id');
query = query.where('ws.deleted = false');
const res = await query.getMany();
return res.map(r => {

6
components/gitpod-protocol/data/gitpod-schema.json
View File

@ -136,9 +136,9 @@
},
"additionalProperties": false
},
"subRepositories": {
"additionalRepositories": {
"type": "array",
"description": "List of sub repositories that are part of this project.",
"description": "List of additional repositories that are part of this project.",
"items": {
"type": "object",
"required": [
@ -157,7 +157,7 @@
"additionalProperties": false
}
},
"mainRepository": {
"mainConfiguration": {
"type": "string",
"description": "The main repository, containing the dev environment configuration."
},

42
components/gitpod-protocol/src/protocol.ts
View File

@ -7,6 +7,7 @@
import { WorkspaceInstance, PortVisibility } from "./workspace-instance";
import { RoleOrPermission } from "./permission";
import { Project } from "./teams-projects-protocol";
import { createHash } from "crypto";
export interface UserInfo {
name?: string
@ -567,14 +568,14 @@ export interface VSCodeConfig {
extensions?: string[];
}
export interface SubRepository {
export interface RepositoryCloneInformation {
url: string;
checkoutLocation?: string;
}
export interface WorkspaceConfig {
mainRepository?: string;
subRepositories?: SubRepository[];
mainConfiguration?: string;
additionalRepositories?: RepositoryCloneInformation[];
image?: ImageConfig;
ports?: PortConfig[];
tasks?: TaskConfig[];
@ -695,6 +696,10 @@ export interface PrebuiltWorkspaceUpdatable {
repo: string;
isResolved: boolean;
installationId: string;
/**
* the commitSHA of the commit that triggered the prebuild
*/
commitSHA?: string;
issue?: string;
contextUrl?: string;
}
@ -875,6 +880,10 @@ export namespace SnapshotContext {
export interface StartPrebuildContext extends WorkspaceContext {
actual: WorkspaceContext;
commitHistory?: string[];
additionalRepositoryCommitHistories?: {
cloneUrl: string;
commitHistory: string[];
}[];
project?: Project;
branch?: string;
}
@ -982,9 +991,32 @@ export interface CommitContext extends WorkspaceContext, GitCheckoutInfo {
cloneUrl?: string
/**
* The clone and checkout information for the sub-repositories in case of multi-repo projects.
* The clone and checkout information for additional repositories in case of multi-repo projects.
*/
subRepositoryCheckoutInfo?: GitCheckoutInfo[];
additionalRepositoryCheckoutInfo?: GitCheckoutInfo[];
}
export namespace CommitContext {
/**
* Creates a hash for all the commits of the CommitContext and all sub-repo commit infos.
* The hash is max 255 chars long.
* @param commitContext
* @returns hash for commitcontext
*/
export function computeHash(commitContext: CommitContext): string {
// for single commits we use the revision to be backward compatible.
if (!commitContext.additionalRepositoryCheckoutInfo || commitContext.additionalRepositoryCheckoutInfo.length === 0) {
return commitContext.revision;
}
const hasher = createHash('sha256');
hasher.update(commitContext.revision);
for (const info of commitContext.additionalRepositoryCheckoutInfo) {
hasher.update(info.revision);
}
return hasher.digest('hex');
}
}
export interface GitCheckoutInfo extends Commit {

4
components/gitpod-protocol/src/wsready.ts
View File

@ -1,10 +1,10 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
// generated using github.com/32leaves/bel on 2021-11-04 12:16:53.917570766 +0000 UTC m=+0.006002884
// generated using github.com/32leaves/bel on 2022-02-15 11:53:18.380158212 +0000 UTC m=+0.011913675
// DO NOT MODIFY
export enum WorkspaceInitSource {

2
components/server/ee/src/container-module.ts
View File

@ -26,7 +26,6 @@ import { BitbucketApp } from "./prebuilds/bitbucket-app";
import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app";
import { IPrefixContextParser } from "../../src/workspace/context-parser";
import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser";
import { StartIncrementalPrebuildContextParser } from "./prebuilds/start-incremental-prebuild-context-parser";
import { WorkspaceFactory } from "../../src/workspace/workspace-factory";
import { WorkspaceFactoryEE } from "./workspace/workspace-factory";
import { MonitoringEndpointsAppEE } from "./monitoring-endpoint-ee";
@ -60,7 +59,6 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
bind(WorkspaceHealthMonitoring).toSelf().inSingletonScope();
bind(PrebuildManager).toSelf().inSingletonScope();
bind(IPrefixContextParser).to(StartPrebuildContextParser).inSingletonScope();
bind(IPrefixContextParser).to(StartIncrementalPrebuildContextParser).inSingletonScope();
bind(GithubApp).toSelf().inSingletonScope();
bind(GitHubAppSupport).toSelf().inSingletonScope();
bind(GithubAppRules).toSelf().inSingletonScope();

30
components/server/ee/src/prebuilds/bitbucket-app.ts
View File

@ -7,10 +7,13 @@
import * as express from 'express';
import { postConstruct, injectable, inject } from 'inversify';
import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib';
import { User, StartPrebuildResult, Project } from '@gitpod/gitpod-protocol';
import { User, StartPrebuildResult, CommitContext, CommitInfo, Project } from '@gitpod/gitpod-protocol';
import { PrebuildManager } from '../prebuilds/prebuild-manager';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { TokenService } from '../../../src/user/token-service';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
@injectable()
export class BitbucketApp {
@ -20,6 +23,8 @@ export class BitbucketApp {
@inject(TokenService) protected readonly tokenService: TokenService;
@inject(ProjectDB) protected readonly projectDB: ProjectDB;
@inject(TeamDB) protected readonly teamDB: TeamDB;
@inject(ContextParser) protected readonly contextParser: ContextParser;
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
protected _router = express.Router();
public static path = '/apps/bitbucket/';
@ -88,25 +93,24 @@ export class BitbucketApp {
const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx);
try {
const contextURL = this.createContextUrl(data);
const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
span.setTag('contextURL', contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
if (!this.prebuildManager.shouldPrebuild(config)) {
console.log('Bitbucket push event: No config. No prebuild.');
return undefined;
}
console.debug('Bitbucket push event: Starting prebuild.', { contextURL });
console.log('Starting prebuild.', { contextURL })
const {host, owner, repo} = RepoURL.parseRepoUrl(data.repoUrl)!;
const hostCtx = this.hostCtxProvider.get(host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx.services.repositoryProvider.getCommitInfo(user, owner, repo, data.commitHash);
}
const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user);
const ws = await this.prebuildManager.startPrebuild({ span }, {
user: projectAndOwner.user,
project: projectAndOwner?.project,
branch: data.branchName,
contextURL,
cloneURL: data.gitCloneUrl,
commit: data.commitHash
});
// todo@alex: add branch and project args
const ws = await this.prebuildManager.startPrebuild({ span }, { user, project: projectAndOwner?.project, context, commitInfo });
return ws;
} finally {
span.finish();

107
components/server/ee/src/prebuilds/github-app.ts
View File

@ -12,13 +12,17 @@ import { Config } from '../../../src/config';
import { AppInstallationDB, TracedWorkspaceDB, DBWithTracing, UserDB, WorkspaceDB, ProjectDB, TeamDB } from '@gitpod/gitpod-db/lib';
import * as express from 'express';
import { log, LogContext, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
import { WorkspaceConfig, User, Project, StartPrebuildResult } from '@gitpod/gitpod-protocol';
import { WorkspaceConfig, User, Project, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol';
import { GithubAppRules } from './github-app-rules';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { PrebuildManager } from './prebuild-manager';
import { PrebuildStatusMaintainer } from './prebuilt-status-maintainer';
import { Options, ApplicationFunctionOptions } from 'probot/lib/types';
import { asyncHandler } from '../../../src/express-util';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
/**
* GitHub app urls:
@ -39,6 +43,8 @@ export class GithubApp {
@inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing<WorkspaceDB>;
@inject(GithubAppRules) protected readonly appRules: GithubAppRules;
@inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager;
@inject(ContextParser) protected readonly contextParser: ContextParser;
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
readonly server: Server | undefined;
@ -159,6 +165,19 @@ export class GithubApp {
});
}
private async findOwnerAndProject(installationID: number | undefined, cloneURL: string): Promise<{ user: User, project?: Project }> {
const installationOwner = installationID ? await this.findInstallationOwner(installationID) : undefined;
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: cloneURL, installationID, project });
throw new Error(`No installation found for ${installationID}`);
}
return {
user, project
}
}
protected async handlePushEvent(ctx: Context<'push'>): Promise<void> {
const span = TraceContext.startSpan("GithubApp.handlePushEvent", {});
span.setTag("request", ctx.id);
@ -166,13 +185,7 @@ export class GithubApp {
try {
const installationId = ctx.payload.installation?.id;
const cloneURL = ctx.payload.repository.clone_url;
const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined;
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: ctx.payload.repository, installationId, project });
return;
}
let { user, project } = await this.findOwnerAndProject(installationId, cloneURL);
const logCtx: LogContext = { userId: user.id };
if (!!user.blocked) {
@ -192,7 +205,13 @@ export class GithubApp {
const contextURL = `${repo.html_url}/tree/${branch}`;
span.setTag('contextURL', contextURL);
let config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
const r = await this.ensureMainProjectAndUser(user, project, context, installationId);
user = r.user;
project = r.project;
const runPrebuild = this.appRules.shouldRunPrebuild(config, branch == repo.default_branch, false, false);
if (!runPrebuild) {
const reason = `Not running prebuild, the user did not enable it for this context`;
@ -201,7 +220,8 @@ export class GithubApp {
return;
}
this.prebuildManager.startPrebuild({ span }, { user, contextURL, cloneURL: repo.clone_url, commit: pl.after, branch, project})
const commitInfo = await this.getCommitInfo(user, repo.html_url, ctx.payload.after);
this.prebuildManager.startPrebuild({ span }, { user, context, project, commitInfo})
.catch(err => log.error(logCtx, "Error while starting prebuild", err, { contextURL }));
} catch (e) {
TraceContext.setError({ span }, e);
@ -211,6 +231,33 @@ export class GithubApp {
}
}
private async ensureMainProjectAndUser(user: User, project: Project | undefined, context: CommitContext, installationId?: number): Promise<{user: User, project?: Project}> {
// if it's a sub-repo of a multi-repo project, we look up the owner of the main repo
if (!!context.additionalRepositoryCheckoutInfo && (!project || context.repository.cloneUrl !== project.cloneUrl)) {
const owner = await this.findOwnerAndProject(installationId, context.repository.cloneUrl);
if (owner) {
return {
user: owner.user,
project: owner.project || project
};
}
}
return {
user,
project
};
}
private async getCommitInfo(user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = this.hostCtxProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
protected getBranchFromRef(ref: string): string | undefined {
const headsPrefix = "refs/heads/";
if (ref.startsWith(headsPrefix)) {
@ -227,22 +274,23 @@ export class GithubApp {
try {
const installationId = ctx.payload.installation?.id;
const cloneURL = ctx.payload.repository.clone_url;
const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined;
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info("Did not find user for installation. Probably an incomplete app installation.", { repo: ctx.payload.repository, installationId, project });
return;
}
const pr = ctx.payload.pull_request;
const contextURL = pr.html_url;
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
let { user, project} = await this.findOwnerAndProject(installationId, cloneURL);
const prebuildStartPromise = this.onPrStartPrebuild({ span }, ctx, config, user, project);
this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise).catch(() => {/** ignore */});
this.onPrAddBadge(config, ctx);
this.onPrAddComment(config, ctx).catch(() => {/** ignore */});
const context = await this.contextParser.handle( { span }, user, contextURL) as CommitContext;
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
const r = await this.ensureMainProjectAndUser(user, project, context, installationId);
user = r.user;
project = r.project;
const prebuildStartPromise = await this.onPrStartPrebuild({ span }, ctx, config, context, user, project);
if (prebuildStartPromise) {
await this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise);
this.onPrAddBadge(config, ctx);
await this.onPrAddComment(config, ctx);
}
} catch (e) {
TraceContext.setError({ span }, e);
throw e;
@ -251,7 +299,7 @@ export class GithubApp {
}
}
protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: Promise<StartPrebuildResult> | undefined) {
protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: StartPrebuildResult) {
if (!start) {
return;
}
@ -262,8 +310,7 @@ export class GithubApp {
const span = TraceContext.startSpan("onPrAddCheck", tracecContext);
try {
const spr = await start;
const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(spr.wsid);
const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(start.wsid);
if (!pws) {
return;
}
@ -286,18 +333,16 @@ export class GithubApp {
}
}
protected onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig | undefined, user: User, project?: Project): Promise<StartPrebuildResult> | undefined {
protected async onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig, context: CommitContext, user: User, project?: Project): Promise<StartPrebuildResult | undefined> {
const pr = ctx.payload.pull_request;
const pr_head = pr.head;
const contextURL = pr.html_url;
const branch = pr.head.ref;
const cloneURL = pr_head.repo.clone_url;
const isFork = pr.head.repo.id !== pr.base.repo.id;
const runPrebuild = this.appRules.shouldRunPrebuild(config, false, true, isFork);
let prebuildStartPromise: Promise<StartPrebuildResult> | undefined;
if (runPrebuild) {
prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, contextURL, cloneURL, commit: pr_head.sha, branch, project});
const commitInfo = await this.getCommitInfo(user, ctx.payload.repository.html_url, pr.head.sha);
prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, context, project, commitInfo});
prebuildStartPromise.catch(err => log.error(err, "Error while starting prebuild", { contextURL }));
return prebuildStartPromise;
} else {

35
components/server/ee/src/prebuilds/gitlab-app.ts
View File

@ -7,13 +7,15 @@
import * as express from 'express';
import { postConstruct, injectable, inject } from 'inversify';
import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib';
import { Project, User, StartPrebuildResult } from '@gitpod/gitpod-protocol';
import { Project, User, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol';
import { PrebuildManager } from '../prebuilds/prebuild-manager';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { TokenService } from '../../../src/user/token-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { GitlabService } from './gitlab-service';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { RepoURL } from '../../../src/repohost';
@injectable()
export class GitLabApp {
@ -24,6 +26,7 @@ export class GitLabApp {
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
@inject(ProjectDB) protected readonly projectDB: ProjectDB;
@inject(TeamDB) protected readonly teamDB: TeamDB;
@inject(ContextParser) protected readonly contextParser: ContextParser;
protected _router = express.Router();
public static path = '/apps/gitlab/';
@ -96,7 +99,9 @@ export class GitLabApp {
const contextURL = this.createContextUrl(body);
log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL });
span.setTag('contextURL', contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
const projectAndOwner = await this.findProjectAndOwner(context.repository.cloneUrl, user);
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
if (!this.prebuildManager.shouldPrebuild(config)) {
log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { context: body, contextURL });
return undefined;
@ -104,18 +109,12 @@ export class GitLabApp {
log.debug({ userId: user.id }, "GitLab push hook: Starting prebuild", { body, contextURL });
const cloneURL = body.repository.git_http_url;
const branch = this.getBranchFromRef(body.ref);
const projectAndOwner = await this.findProjectAndOwner(cloneURL, user);
const commitInfo = await this.getCommitInfo(user, body.repository.git_http_url, body.after);
const ws = await this.prebuildManager.startPrebuild({ span }, {
user: projectAndOwner.user,
user: projectAndOwner?.user || user,
project: projectAndOwner?.project,
contextURL,
cloneURL,
commit: body.after,
branch,
context,
commitInfo
});
return ws;
@ -124,6 +123,16 @@ export class GitLabApp {
}
}
private async getCommitInfo(user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = this.hostCtxProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
/**
* Finds the relevant user account and project to the provided webhook event information.
*
@ -135,7 +144,7 @@ export class GitLabApp {
* @param webhookInstaller the user account known from the webhook installation
* @returns a promise which resolves to a user account and an optional project.
*/
protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> {
protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> {
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
if (project) {
if (project.userId) {

152
components/server/ee/src/prebuilds/prebuild-manager.ts
View File

@ -5,10 +5,10 @@
*/
import { DBWithTracing, TracedWorkspaceDB, WorkspaceDB } from '@gitpod/gitpod-db/lib';
import { CommitContext, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol';
import { CommitContext, CommitInfo, PrebuiltWorkspace, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, Workspace, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { getCommitInfo, HostContextProvider } from '../../../src/auth/host-context-provider';
import { WorkspaceFactory } from '../../../src/workspace/workspace-factory';
import { ConfigProvider } from '../../../src/workspace/config-provider';
import { WorkspaceStarter } from '../../../src/workspace/workspace-starter';
@ -18,7 +18,6 @@ import { secondsBefore } from '@gitpod/gitpod-protocol/lib/util/timeutil';
import { inject, injectable } from 'inversify';
import * as opentracing from 'opentracing';
import { URL } from 'url';
export class WorkspaceRunningError extends Error {
constructor(msg: string, public instance: WorkspaceInstance) {
@ -28,11 +27,9 @@ export class WorkspaceRunningError extends Error {
export interface StartPrebuildParams {
user: User;
contextURL: string;
cloneURL: string;
branch?: string;
commit: string;
context: CommitContext;
project?: Project;
commitInfo?: CommitInfo;
}
const PREBUILD_LIMITER_WINDOW_SECONDS = 60;
@ -48,43 +45,25 @@ export class PrebuildManager {
@inject(Config) protected readonly config: Config;
@inject(ProjectsService) protected readonly projectService: ProjectsService;
async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise<boolean> {
const span = TraceContext.startSpan("hasPrebuilds", ctx);
span.setTag(cloneURL, cloneURL);
try {
const existingPBs = await this.workspaceDB.trace({ span }).findPrebuildsWithWorkpace(cloneURL);
for (const pb of existingPBs) {
if (!pb.workspace.contextURL.startsWith('prebuild')) {
return true;
}
}
return false;
} catch (err) {
TraceContext.setError({ span }, err);
throw err;
} finally {
span.finish();
}
}
async startPrebuild(ctx: TraceContext, { contextURL, cloneURL, commit, branch, project, user }: StartPrebuildParams): Promise<StartPrebuildResult> {
async startPrebuild(ctx: TraceContext, { context, project, user, commitInfo }: StartPrebuildParams): Promise<StartPrebuildResult> {
const span = TraceContext.startSpan("startPrebuild", ctx);
span.setTag("contextURL", contextURL);
const cloneURL = context.repository.cloneUrl;
const commitSHAIdentifier = CommitContext.computeHash(context);
span.setTag("cloneURL", cloneURL);
span.setTag("commit", commit);
span.setTag("commit", commitInfo?.sha);
try {
if (user.blocked) {
throw new Error("Blocked users cannot start prebuilds.");
}
const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commit);
const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commitSHAIdentifier);
// If the existing prebuild is failed, we want to retrigger it.
if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'failed' && existingPB.state !== 'timeout') {
// If the existing prebuild is based on an outdated project config, we also want to retrigger it.
const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId);
const existingConfig = existingPBWS?.config;
const newConfig = await this.fetchConfig({ span }, user, contextURL);
log.debug(`startPrebuild | commit: ${commit}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`);
const newConfig = await this.fetchConfig({ span }, user, context);
log.debug(`startPrebuild | commits: ${commitSHAIdentifier}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`);
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key))
@ -98,35 +77,46 @@ export class PrebuildManager {
}
}
const contextParser = this.getContextParserFor(contextURL);
if (!contextParser) {
throw new Error(`Cannot find context parser for URL: ${contextURL}`);
}
const actual = await contextParser.handle({ span }, user, contextURL) as CommitContext;
actual.revision = commit; // Make sure we target the correct commit here (might have changed between trigger and contextParser lookup)
actual.ref = undefined;
actual.forceCreateNewWorkspace = true;
const prebuildContext: StartPrebuildContext = {
title: `Prebuild of "${actual.title}"`,
actual,
title: `Prebuild of "${context.title}"`,
actual: context,
project,
branch,
normalizedContextURL: actual.normalizedContextURL
branch: context.ref,
normalizedContextURL: context.normalizedContextURL
};
if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl, project)) {
if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) {
const maxDepth = this.config.incrementalPrebuilds.commitHistory;
prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth);
const hostContext = this.hostContextProvider.get(context.repository.host);
const repoProvider = hostContext?.services?.repositoryProvider;
if (repoProvider) {
prebuildContext.commitHistory = await repoProvider.getCommitHistory(user, context.repository.owner, context.repository.name, context.revision, maxDepth);
if (context.additionalRepositoryCheckoutInfo && context.additionalRepositoryCheckoutInfo.length > 0) {
const histories = context.additionalRepositoryCheckoutInfo.map(async info => {
const commitHistory = await repoProvider.getCommitHistory(user, info.repository.owner, info.repository.name, info.revision, maxDepth);
return {
cloneUrl: info.repository.cloneUrl,
commitHistory
}
});
prebuildContext.additionalRepositoryCommitHistories = await Promise.all(histories);
}
}
}
const projectEnvVarsPromise = project ? this.projectService.getProjectEnvironmentVariables(project.id) : [];
const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL);
const prebuild = await this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!;
const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, context.normalizedContextURL!);
const prebuildPromise = this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!;
span.setTag("starting", true);
const projectEnvVars = await projectEnvVarsPromise;
await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']});
const prebuild = await prebuildPromise;
if (!prebuild) {
throw new Error(`Failed to create a prebuild for: ${contextURL}`);
throw new Error(`Failed to create a prebuild for: ${context.normalizedContextURL}`);
}
if (await this.shouldRateLimitPrebuild(span, cloneURL)) {
@ -143,9 +133,20 @@ export class PrebuildManager {
};
}
span.setTag("starting", true);
const projectEnvVars = await projectEnvVarsPromise;
await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']});
if (project) {
let aCommitInfo = commitInfo;
if (!aCommitInfo) {
aCommitInfo = await getCommitInfo(this.hostContextProvider, user, context.repository.cloneUrl, context.revision);
if (!aCommitInfo) {
aCommitInfo = {
author: 'unknown',
commitMessage: 'unknown',
sha: context.revision
}
}
}
await this.storePrebuildInfo({ span }, project, prebuild, workspace, user, aCommitInfo);
}
return { prebuildId: prebuild.id, wsid: workspace.id, done: false };
} catch (err) {
TraceContext.setError({ span }, err);
@ -214,17 +215,10 @@ export class PrebuildManager {
return this.config.incrementalPrebuilds.repositoryPasslist.some(url => trimRepoUrl(url) === repoUrl);
}
async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise<WorkspaceConfig | undefined> {
async fetchConfig(ctx: TraceContext, user: User, context: CommitContext): Promise<WorkspaceConfig> {
const span = TraceContext.startSpan("fetchConfig", ctx);
span.setTag("contextURL", contextURL);
try {
const contextParser = this.getContextParserFor(contextURL);
if (!contextParser) {
return undefined;
}
const context = await contextParser!.handle({ span }, user, contextURL);
return (await this.configProvider.fetchConfig({ span }, user, context as CommitContext)).config;
return (await this.configProvider.fetchConfig({ span }, user, context)).config;
} catch (err) {
TraceContext.setError({ span }, err);
throw err;
@ -233,13 +227,31 @@ export class PrebuildManager {
}
}
protected getContextParserFor(contextURL: string) {
const host = new URL(contextURL).hostname;
const hostContext = this.hostContextProvider.get(host);
if (!hostContext) {
return undefined;
}
return hostContext.contextParser;
//TODO this doesn't belong so deep here. All this context should be stored on the surface not passed down.
protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User, commit: CommitInfo) {
const span = TraceContext.startSpan("storePrebuildInfo", ctx);
const { userId, teamId, name: projectName, id: projectId } = project;
await this.workspaceDB.trace({span}).storePrebuildInfo({
id: pws.id,
buildWorkspaceId: pws.buildWorkspaceId,
basedOnPrebuildId: ws.basedOnPrebuildId,
teamId,
userId,
projectName,
projectId,
startedAt: pws.creationTime,
startedBy: "", // TODO
startedByAvatar: "", // TODO
cloneUrl: pws.cloneURL,
branch: pws.branch || "unknown",
changeAuthor: commit.author,
changeAuthorAvatar: commit.authorAvatarUrl,
changeDate: commit.authorDate || "",
changeHash: commit.sha,
changeTitle: commit.commitMessage,
// changePR
changeUrl: ws.contextURL,
});
}
private async shouldRateLimitPrebuild(span: opentracing.Span, cloneURL: string): Promise<boolean> {
@ -270,4 +282,4 @@ export class PrebuildManager {
// Last resort default
return PREBUILD_LIMITER_DEFAULT_LIMIT;
}
}
}

35
components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts
View File

@ -46,7 +46,7 @@ export class PrebuildStatusMaintainer implements Disposable {
this.disposables.push(
repeat(this.periodicUpdatableCheck.bind(this), 60 * 1000)
);
log.debug("prebuild updatatable status maintainer started");
log.debug("prebuild updatable status maintainer started");
}
public async registerCheckRun(ctx: TraceContext, installationId: number, pws: PrebuiltWorkspace, cri: CheckRunInfo, config?: WorkspaceConfig) {
@ -64,6 +64,7 @@ export class PrebuildStatusMaintainer implements Disposable {
id: uuidv4(),
owner: cri.owner,
repo: cri.repo,
commitSHA: cri.head_sha,
isResolved: false,
installationId: installationId.toString(),
contextUrl: cri.details_url,
@ -131,8 +132,8 @@ export class PrebuildStatusMaintainer implements Disposable {
return;
}
const updatatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id);
await Promise.all(updatatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild)));
const updatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id);
await Promise.all(updatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild)));
} catch (err) {
TraceContext.setError({span}, err);
throw err;
@ -141,38 +142,38 @@ export class PrebuildStatusMaintainer implements Disposable {
}
}
protected async doUpdate(ctx: TraceContext, updatatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise<void> {
protected async doUpdate(ctx: TraceContext, updatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise<void> {
const span = TraceContext.startSpan("doUpdate", ctx);
try {
const githubApi = await this.getGitHubApi(Number.parseInt(updatatable.installationId));
const githubApi = await this.getGitHubApi(Number.parseInt(updatable.installationId));
if (!githubApi) {
log.error("unable to authenticate GitHub app - this leaves user-facing checks dangling.");
return;
}
const workspace = await this.workspaceDB.trace({span}).findById(pws.buildWorkspaceId);
if (!!updatatable.contextUrl && !!workspace) {
if (!!updatable.contextUrl && !!workspace) {
const conclusion = this.getConclusionFromPrebuildState(pws);
if (conclusion === 'pending') {
log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatatable.prebuiltWorkspaceId });
log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatable.prebuiltWorkspaceId });
return;
}
let found = true;
try {
await githubApi.repos.createCommitStatus({
owner: updatatable.owner,
repo: updatatable.repo,
owner: updatable.owner,
repo: updatable.repo,
context: "Gitpod",
sha: pws.commit,
target_url: updatatable.contextUrl,
description: conclusion === 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION,
sha: updatable.commitSHA || pws.commit,
target_url: updatable.contextUrl,
description: conclusion == 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION,
state: (workspace?.config?.github?.prebuilds?.addCheck === 'prevent-merge-on-error' ? conclusion : 'success')
});
} catch (err) {
if (err.message == "Not Found") {
log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatatable.owner, repo: updatatable.repo});
log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatable.owner, repo: updatable.repo});
found = true;
} else {
throw err;
@ -185,10 +186,10 @@ export class PrebuildStatusMaintainer implements Disposable {
},
});
await this.workspaceDB.trace({span}).markUpdatableResolved(updatatable.id);
log.info(`Resolved updatable. Marked check on ${updatatable.contextUrl} as ${conclusion}`);
} else if (!!updatatable.issue) {
// this updatatable updates a label
await this.workspaceDB.trace({span}).markUpdatableResolved(updatable.id);
log.info(`Resolved updatable. Marked check on ${updatable.contextUrl} as ${conclusion}`);
} else if (!!updatable.issue) {
// this updatable updates a label
log.debug("Update label on a PR - we're not using this yet");
}
} catch (err) {

42
components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts
View File

@ -1,42 +0,0 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Licensed under the Gitpod Enterprise Source Code License,
* See License.enterprise.txt in the project root folder.
*/
import { User, WorkspaceContext, StartPrebuildContext, CommitContext, ContextURL } from "@gitpod/gitpod-protocol";
import { inject, injectable } from "inversify";
import { URL } from "url";
import { Config } from '../../../src/config';
import { HostContextProvider } from "../../../src/auth/host-context-provider";
import { IPrefixContextParser } from "../../../src/workspace/context-parser";
@injectable()
export class StartIncrementalPrebuildContextParser implements IPrefixContextParser {
@inject(Config) protected readonly config: Config;
@inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
static PREFIX = ContextURL.INCREMENTAL_PREBUILD_PREFIX + '/';
findPrefix(user: User, context: string): string | undefined {
if (context.startsWith(StartIncrementalPrebuildContextParser.PREFIX)) {
return StartIncrementalPrebuildContextParser.PREFIX;
}
}
public async handle(user: User, prefix: string, context: WorkspaceContext): Promise<WorkspaceContext> {
if (!CommitContext.is(context)) {
throw new Error("can only start incremental prebuilds on a commit context")
}
const host = new URL(context.repository.cloneUrl).hostname;
const hostContext = this.hostContextProvider.get(host);
const maxDepth = this.config.incrementalPrebuilds.commitHistory;
const result: StartPrebuildContext = {
title: `Prebuild of "${context.title}"`,
actual: context,
commitHistory: await (hostContext?.contextParser?.fetchCommitHistory({}, user, context.repository.cloneUrl, context.revision, maxDepth) || [])
};
return result;
}
}

37
components/server/ee/src/workspace/gitpod-server-impl.ts
View File

@ -661,18 +661,19 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
return;
}
const commitSHAs = CommitContext.computeHash(context);
const logCtx: LogContext = { userId: user.id };
const cloneUrl = context.repository.cloneUrl;
// Note: findPrebuiltWorkspaceByCommit always returns the last triggered prebuild (so, if you re-trigger a prebuild, the newer one will always be used here)
const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision);
const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace };
const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs);
const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace };
log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload);
if (!prebuiltWorkspace) {
return;
}
if (prebuiltWorkspace.state === 'available') {
log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${context.revision}`, logPayload);
log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${commitSHAs}`, logPayload);
const result: PrebuiltWorkspaceContext = {
title: context.title,
originalContext: context,
@ -752,7 +753,7 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
} else {
result.runningWorkspacePrebuild!.starting = 'running';
}
log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${context.revision}`, logPayload);
log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${commitSHAs}`, logPayload);
return result;
}
}
@ -766,27 +767,6 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
await this.licenseEvaluator.reloadLicense();
}
// TODO(gpl) This is not part of our API interface, nor can I find any clients. Remove or re-surrect?
// async getLicenseInfo(ctx: TraceContext): Promise<GetLicenseInfoResult> {
// const user = this.checkAndBlockUser("getLicenseInfo");
// const { key } = await this.licenseKeySource.getKey();
// const { validUntil, seats } = this.licenseEvaluator.inspect();
// const { valid } = this.licenseEvaluator.validate();
// const isAdmin = this.authorizationService.hasPermission(user, Permission.ADMIN_API);
// return {
// isAdmin,
// licenseInfo: {
// key: isAdmin ? key : "REDACTED",
// seats,
// valid,
// validUntil
// }
// };
// }
async licenseIncludesFeature(ctx: TraceContext, licenseFeature: LicenseFeature): Promise<boolean> {
traceAPIParams(ctx, { licenseFeature });
@ -1541,11 +1521,8 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
const context = await this.contextParser.handle(ctx, user, contextURL) as CommitContext;
const prebuild = await this.prebuildManager.startPrebuild(ctx, {
contextURL,
cloneURL: project.cloneUrl,
commit: context.revision,
context,
user,
branch: branchDetails[0].name,
project
});

179
components/server/ee/src/workspace/workspace-factory.ts
View File

@ -8,14 +8,13 @@ import { v4 as uuidv4 } from 'uuid';
import { WorkspaceFactory } from "../../../src/workspace/workspace-factory";
import { injectable, inject } from "inversify";
import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, Project, PrebuiltWorkspace } from "@gitpod/gitpod-protocol";
import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, PrebuiltWorkspace, WorkspaceConfig, WorkspaceImageSource } from "@gitpod/gitpod-protocol";
import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { LicenseEvaluator } from '@gitpod/licensor/lib';
import { Feature } from '@gitpod/licensor/lib/api';
import { ResponseError } from 'vscode-jsonrpc';
import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
@injectable()
export class WorkspaceFactoryEE extends WorkspaceFactory {
@ -51,7 +50,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
const { project, branch } = context;
const commitContext: CommitContext = context.actual;
const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, commitContext.revision);
const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, CommitContext.computeHash(commitContext));
if (existingPWS) {
const wsInstance = await this.db.trace({span}).findRunningInstance(existingPWS.buildWorkspaceId);
if (wsInstance) {
@ -62,62 +61,28 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
const { config } = await this.configProvider.fetchConfig({span}, user, context.actual);
const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config);
// Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on.
// Walk back the last prebuilds and check if they are valid ancestor.
let ws;
for (const parent of (context.commitHistory || [])) {
const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent);
if (!parentPrebuild) {
continue;
if (context.commitHistory && context.commitHistory.length > 0) {
const recentPrebuilds = await this.db.trace({span}).findPrebuildsWithWorkpace(commitContext.repository.cloneUrl);
const match = recentPrebuilds.find(pb => this.isGoodBaseforIncrementalPrebuild(context, config, imageSource, pb.prebuild, pb.workspace));
if (match) {
const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
title: `Incremental prebuild of "${commitContext.title}"`,
originalContext: commitContext,
prebuiltWorkspace: match.prebuild,
}
ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
// Overwrite the config from the parent prebuild:
// `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild.
// Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now.
// However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...)
// To fix this, we overwrite the new prebuild's config with the most-recently fetched config.
// See also: https://github.com/gitpod-io/gitpod/issues/7475
//TODO(sven) doing side effects on objects back and forth is complicated and error-prone. We should rather make sure we pass in the config when creating the prebuiltWorkspace.
ws.config = config;
}
if (parentPrebuild.state !== 'available') {
continue;
}
log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild);
const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId);
if (!buildWorkspace) {
continue;
}
if (!!buildWorkspace.basedOnPrebuildId) {
continue;
}
if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) {
log.debug(`Skipping parent prebuild: Outdated image`, {
imageSource,
parentImageSource: buildWorkspace.imageSource,
});
continue;
}
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key))
// @ts-ignore
.reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
.filter(task => Object.keys(task).length > 0));
const prebuildTasks = filterPrebuildTasks(config.tasks);
const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks);
if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
prebuildTasks,
parentPrebuildTasks,
});
continue;
}
const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
title: `Incremental prebuild of "${commitContext.title}"`,
originalContext: commitContext,
prebuiltWorkspace: parentPrebuild,
}
ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
// Overwrite the config from the parent prebuild:
// `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild.
// Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now.
// However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...)
// To fix this, we overwrite the new prebuild's config with the most-recently fetched config.
// See also: https://github.com/gitpod-io/gitpod/issues/7475
ws.config = config;
break;
}
if (!ws) {
// No suitable parent prebuild was found -- create a (fresh) full prebuild.
ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
@ -130,21 +95,13 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
id: uuidv4(),
buildWorkspaceId: ws.id,
cloneURL: commitContext.repository.cloneUrl,
commit: commitContext.revision,
commit: CommitContext.computeHash(commitContext),
state: "queued",
creationTime: new Date().toISOString(),
projectId: ws.projectId,
branch
});
if (project) {
// do not await
this.storePrebuildInfo(ctx, project, pws, ws, user).catch(err => {
log.error(`failed to store prebuild info`, err);
TraceContext.setError({span}, err);
});
}
log.debug({ userId: user.id, workspaceId: ws.id }, `Registered workspace prebuild: ${pws.id} for ${commitContext.repository.cloneUrl}:${commitContext.revision}`);
return ws;
@ -156,43 +113,67 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
}
}
protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User) {
const span = TraceContext.startSpan("storePrebuildInfo", ctx);
const { userId, teamId, name: projectName, id: projectId } = project;
const parsedUrl = RepoURL.parseRepoUrl(project.cloneUrl);
if (!parsedUrl) {
return;
private async isGoodBaseforIncrementalPrebuild(context: StartPrebuildContext, config: WorkspaceConfig, imageSource: WorkspaceImageSource, candidatePrebuild: PrebuiltWorkspace, candidate: Workspace) {
if (!context.commitHistory || context.commitHistory.length === 0) {
return false;
}
const { owner, repo, host } = parsedUrl;
const repositoryProvider = this.hostContextProvider.get(host)?.services?.repositoryProvider;
if (!repositoryProvider) {
return;
if (!CommitContext.is(candidate.context)) {
return false;
}
const commit = await repositoryProvider.getCommitInfo(user, owner, repo, pws.commit);
if (!commit) {
return;
// we are only considering available prebuilds
if (candidatePrebuild.state !== 'available') {
return false;
}
// we are only considering full prebuilds
if (!!candidate.basedOnPrebuildId) {
return false;
}
const candidateCtx = candidate.context;
if (candidateCtx.additionalRepositoryCheckoutInfo?.length !== context.additionalRepositoryCommitHistories?.length) {
// different number of repos
return false;
}
if (!context.commitHistory.some(sha => sha === candidateCtx.revision)) {
return false;
}
// check the commits are included in the commit history
for (const subRepo of candidateCtx.additionalRepositoryCheckoutInfo || []) {
const matchIngRepo = context.additionalRepositoryCommitHistories?.find(repo => repo.cloneUrl === subRepo.repository.cloneUrl);
if (!matchIngRepo || !matchIngRepo.commitHistory.some(sha => sha === subRepo.revision)) {
return false;
}
}
// ensure the image source hasn't changed
if (JSON.stringify(imageSource) !== JSON.stringify(candidate.imageSource)) {
log.debug(`Skipping parent prebuild: Outdated image`, {
imageSource,
parentImageSource: candidate.imageSource,
});
return false;
}
// ensure the tasks haven't changed
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key))
// @ts-ignore
.reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
.filter(task => Object.keys(task).length > 0));
const prebuildTasks = filterPrebuildTasks(config.tasks);
const parentPrebuildTasks = filterPrebuildTasks(candidate.config.tasks);
if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
prebuildTasks,
parentPrebuildTasks,
});
return false;
}
await this.db.trace({span}).storePrebuildInfo({
id: pws.id,
buildWorkspaceId: pws.buildWorkspaceId,
basedOnPrebuildId: ws.basedOnPrebuildId,
teamId,
userId,
projectName,
projectId,
startedAt: pws.creationTime,
startedBy: "", // TODO
startedByAvatar: "", // TODO
cloneUrl: pws.cloneURL,
branch: pws.branch || "unknown",
changeAuthor: commit.author,
changeAuthorAvatar: commit.authorAvatarUrl,
changeDate: commit.authorDate || "",
changeHash: commit.sha,
changeTitle: commit.commitMessage,
// changePR
changeUrl: ws.contextURL,
});
}
protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise<Workspace> {

2
components/server/install-gh-app.sh
View File

@ -3,7 +3,6 @@
# This script will patch the servers config map, install the app cert and restart the server components
# It is best to add the envs to your environment variables using `gp env GH_APP_ID=....` and `gp env GH_APP_KEY="..."`.
# See https://www.notion.so/gitpod/How-to-deploy-a-PR-with-a-working-GitHub-App-integration-d297a1ef2f7b4b3aa8483b2ae9b47da2 (internal) for more details.
# GH_APP_ID=<app-id>
# GH_APP_KEY="-----BEGIN RSA PRIVATE KEY-----
# ...
@ -32,6 +31,7 @@ kubectl get cm server-config -o yaml > server-config.yml
perl -0777 -i.original -pe "s/\"githubApp\":.+?\}/$LINE/igs" server-config.yml
kubectl apply -f server-config.yml
rm server-config.yml
rm server-config.yml.original
echo 'updating the secret'
kubectl delete secret server-github-app-cert

11
components/server/src/auth/host-context-provider.ts
View File

@ -6,6 +6,8 @@
import { HostContext } from "./host-context";
import { AuthProviderParams } from "./auth-provider";
import { CommitInfo, User } from "@gitpod/gitpod-protocol";
import { RepoURL } from "../repohost";
export const HostContextProvider = Symbol("HostContextProvider");
@ -16,6 +18,15 @@ export interface HostContextProvider {
findByAuthProviderId(authProviderId: string): HostContext | undefined;
}
export async function getCommitInfo(hostContextProvider: HostContextProvider, user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = hostContextProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
export const HostContextProviderFactory = Symbol("HostContextProviderFactory");

59
components/server/src/auth/resource-access.ts
View File

@ -4,8 +4,10 @@
* See License-AGPL.txt in the project root for license information.
*/
import { CommitContext, ContextURL, GitpodToken, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
import { CommitContext, GitpodToken, Repository, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
import { UnauthorizedError } from "../errors";
import { RepoURL } from "../repohost";
import { HostContextProvider } from "./host-context-provider";
declare var resourceInstance: GuardedResource;
@ -477,27 +479,40 @@ export class RepositoryResourceGuard implements ResourceAccessGuard {
// Check if user has at least read access to the repository
const workspace = resource.kind === 'snapshot' ? resource.workspace : resource.subject;
const contextURL = ContextURL.getNormalizedURL(workspace);
if (!contextURL) {
throw new Error(`unable to parse ContextURL: ${contextURL}`);
const repos: Repository[] = [];
if (CommitContext.is(workspace.context)) {
repos.push(workspace.context.repository);
for (const additionalRepo of workspace.context.additionalRepositoryCheckoutInfo || []) {
repos.push(additionalRepo.repository);
}
}
const hostContext = this.hostContextProvider.get(contextURL.hostname);
if (!hostContext) {
throw new Error(`no HostContext found for hostname: ${contextURL.hostname}`);
}
const { authProvider } = hostContext;
const identity = User.getIdentity(this.user, authProvider.authProviderId);
if (!identity) {
throw UnauthorizedError.create(contextURL.hostname, authProvider.info.requirements?.default || [], "missing-identity");
}
const { services } = hostContext;
if (!services) {
throw new Error(`no services found in HostContext for hostname: ${contextURL.hostname}`);
}
if (!CommitContext.is(workspace.context)) {
return false;
}
const { owner, name: repo } = workspace.context.repository;
return services.repositoryProvider.hasReadAccess(this.user, owner, repo);
const result = await Promise.all(
repos.map(
async repo => {
const repoUrl = RepoURL.parseRepoUrl(repo.cloneUrl);
if (!repoUrl) {
log.error("Cannot parse repoURL", {repo})
return false;
}
const hostContext = this.hostContextProvider.get(repoUrl.host)
if (!hostContext) {
throw new Error(`no HostContext found for hostname: ${repoUrl.host}`);
}
const { authProvider } = hostContext;
const identity = User.getIdentity(this.user, authProvider.authProviderId);
if (!identity) {
throw UnauthorizedError.create(repoUrl!.host, authProvider.info.requirements?.default || [], "missing-identity");
}
const { services } = hostContext;
if (!services) {
throw new Error(`no services found in HostContext for hostname: ${repoUrl.host}`);
}
if (!CommitContext.is(workspace.context)) {
return false;
}
return services.repositoryProvider.hasReadAccess(this.user, repo.owner, repo.name);
}
));
return result.every(b => b);
}
}

3
components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts
View File

@ -107,4 +107,7 @@ export class BitbucketServerRepositoryProvider implements RepositoryProvider {
return false;
}
async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise<string[]> {
return [];
}
}

7
components/server/src/bitbucket/bitbucket-context-parser.spec.ts
View File

@ -471,13 +471,6 @@ class TestBitbucketContextParser {
"title": "gitpod/integration-tests-forked-repository - master"
})
}
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
expect(result).to.deep.equal([
'da2119f51b0e744cb6b36399f8433b477a4174ef',
])
}
}
module.exports = new TestBitbucketContextParser();

27
components/server/src/bitbucket/bitbucket-context-parser.ts
View File

@ -267,31 +267,4 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo
return result;
}
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[] | undefined> {
const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx);
try {
// TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
// The additional property 'page' may be helfpul.
const api = await this.api(user);
const { owner, repoName } = await this.parseURL(user, contextUrl);
const result = await api.repositories.listCommitsAt({
workspace: owner,
repo_slug: repoName,
revision: sha,
pagelen: maxDepth,
});
const commits = result.data.values?.slice(1);
if (!commits) {
return undefined;
}
return commits.map((v: Schema.Commit) => v.hash!);
} catch (e) {
span.log({ error: e });
log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e);
throw e;
} finally {
span.finish();
}
}
}

6
components/server/src/bitbucket/bitbucket-repository-provider.spec.ts
View File

@ -72,6 +72,12 @@ class TestBitbucketRepositoryProvider {
});
}
@test public async testFetchCommitHistory() {
const result = await this.repoProvider.getCommitHistory(this.user, 'gitpod', 'integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
expect(result).to.deep.equal([
'da2119f51b0e744cb6b36399f8433b477a4174ef',
])
}
}
module.exports = new TestBitbucketRepositoryProvider();

19
components/server/src/bitbucket/bitbucket-repository-provider.ts
View File

@ -5,6 +5,7 @@
*/
import { Branch, CommitInfo, Repository, User } from "@gitpod/gitpod-protocol";
import { Schema } from "bitbucket";
import { inject, injectable } from 'inversify';
import { URL } from "url";
import { RepoURL } from '../repohost/repo-url';
@ -108,4 +109,22 @@ export class BitbucketRepositoryProvider implements RepositoryProvider {
// FIXME(janx): Not implemented yet
return false;
}
public async getCommitHistory(user: User, owner: string, repo: string, revision: string, maxDepth: number = 100): Promise<string[]> {
const api = await this.apiFactory.create(user);
// TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
// The additional property 'page' may be helfpul.
const result = await api.repositories.listCommitsAt({
workspace: owner,
repo_slug: repo,
revision: revision,
pagelen: maxDepth,
});
const commits = result.data.values?.slice(1);
if (!commits) {
return [];
}
return commits.map((v: Schema.Commit) => v.hash!);
}
}

2
components/server/src/github/github-auth-provider.ts
View File

@ -69,7 +69,7 @@ export class GitHubAuthProvider extends GenericAuthProvider {
timeout: 5000,
},
userAgent: this.USER_AGENT,
baseUrl: this.baseURL
baseUrl: this.baseURL,
});
const fetchCurrentUser = async () => {
const response = await api.users.getAuthenticated();

9
components/server/src/github/github-context-parser.spec.ts
View File

@ -577,14 +577,5 @@ class TestGithubContextParser {
}
)
}
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://github.com/gitpod-io/gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100);
expect(result).to.deep.equal([
'506e5aed317f28023994ecf8ca6ed91430e9c1a4',
'f5b041513bfab914b5fbf7ae55788d9835004d76',
])
}
}
module.exports = new TestGithubContextParser() // Only to circumvent no usage warning :-/

51
components/server/src/github/github-context-parser.ts
View File

@ -426,55 +426,4 @@ export class GithubContextParser extends AbstractContextParser implements IConte
`;
}
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
const span = TraceContext.startSpan("GithubContextParser.fetchCommitHistory", ctx);
try {
if (sha.length != 40) {
throw new Error(`Invalid commit ID ${sha}.`);
}
// TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
// These additional history properties may be helfpul:
// totalCount,
// pageInfo {
// haxNextPage,
// },
const { owner, repoName } = await this.parseURL(user, contextUrl);
const result: any = await this.githubQueryApi.runQuery(user, `
query {
repository(name: "${repoName}", owner: "${owner}") {
object(oid: "${sha}") {
... on Commit {
history(first: ${maxDepth}) {
edges {
node {
oid
}
}
}
}
}
}
}
`);
span.log({"request.finished": ""});
if (result.data.repository === null) {
throw await NotFoundError.create(await this.tokenHelper.getCurrentToken(user), user, this.config.host, owner, repoName);
}
const commit = result.data.repository.object;
if (commit === null) {
throw new Error(`Couldn't find commit ${sha} in repository ${owner}/${repoName}.`);
}
return commit.history.edges.slice(1).map((e: any) => e.node.oid) || [];
} catch (e) {
span.log({"error": e});
throw e;
} finally {
span.finish();
}
}
}

76
components/server/src/github/github-repository-provider.spec.ts Normal file
View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2020 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
// Use asyncIterators with es2015
if (typeof (Symbol as any).asyncIterator === 'undefined') {
(Symbol as any).asyncIterator = Symbol.asyncIterator || Symbol('asyncIterator');
}
import "reflect-metadata";
import { suite, test, timeout, retries } from "mocha-typescript";
import * as chai from 'chai';
const expect = chai.expect;
import { GitHubGraphQlEndpoint, GitHubRestApi } from './api';
import { User } from "@gitpod/gitpod-protocol";
import { ContainerModule, Container } from "inversify";
import { Config } from "../config";
import { DevData } from "../dev/dev-data";
import { AuthProviderParams } from "../auth/auth-provider";
import { TokenProvider } from "../user/token-provider";
import { GitHubTokenHelper } from "./github-token-helper";
import { HostContextProvider } from "../auth/host-context-provider";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { GithubRepositoryProvider } from "./github-repository-provider";
@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITHUB"))
class TestGithubContextRepositoryProvider {
protected provider: GithubRepositoryProvider;
protected user: User;
public before() {
const container = new Container();
container.load(new ContainerModule((bind, unbind, isBound, rebind) => {
bind(Config).toConstantValue({
// meant to appease DI, but Config is never actually used here
});
bind(GithubRepositoryProvider).toSelf().inSingletonScope();
bind(GitHubRestApi).toSelf().inSingletonScope();
bind(GitHubGraphQlEndpoint).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestGithubContextRepositoryProvider.AUTH_HOST_CONFIG);
bind(GitHubTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async (user: User, host: string) => {
return DevData.createGitHubTestToken();
}
});
bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider());
}));
this.provider = container.get(GithubRepositoryProvider);
this.user = DevData.createTestUser();
}
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitHub",
type: "GitHub",
verified: true,
description: "",
icon: "",
host: "github.com",
oauth: "not-used" as any
}
@test public async testFetchCommitHistory() {
const result = await this.provider.getCommitHistory(this.user, 'gitpod-io', 'gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100);
expect(result).to.deep.equal([
'506e5aed317f28023994ecf8ca6ed91430e9c1a4',
'f5b041513bfab914b5fbf7ae55788d9835004d76',
])
}
}
module.exports = new TestGithubContextRepositoryProvider() // Only to circumvent no usage warning :-/

54
components/server/src/github/github-repository-provider.ts
View File

@ -103,8 +103,58 @@ export class GithubRepositoryProvider implements RepositoryProvider {
}
async getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined> {
const commit = await this.github.getCommit(user, { repo, owner, ref });
return commit;
try {
return await this.github.getCommit(user, { repo, owner, ref });
} catch (error) {
console.error(error);
return undefined;
}
}
public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise<string[]> {
try {
if (ref.length != 40) {
throw new Error(`Invalid commit ID ${ref}.`);
}
// TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
// These additional history properties may be helfpul:
// totalCount,
// pageInfo {
// haxNextPage,
// },
const result: any = await this.githubQueryApi.runQuery(user, `
query {
repository(name: "${repo}", owner: "${owner}") {
object(oid: "${ref}") {
... on Commit {
history(first: ${maxDepth}) {
edges {
node {
oid
}
}
}
}
}
}
}
`);
if (result.data.repository === null) {
throw new Error(`couldn't find repository ${owner}/${repo} on ${this.github.baseURL}`);
}
const commit = result.data.repository.object;
if (commit === null) {
throw new Error(`Couldn't find commit ${ref} in repository ${owner}/${repo}.`);
}
return commit.history.edges.slice(1).map((e: any) => e.node.oid) || [];
} catch (e) {
console.error(e);
return [];
}
}
async getUserRepos(user: User): Promise<string[]> {

8
components/server/src/gitlab/gitlab-context-parser.spec.ts
View File

@ -604,14 +604,6 @@ class TestGitlabContextParser {
})
}
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://gitlab.com/AlexTugarev/gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100);
expect(result).to.deep.equal([
'4447fbc4d46e6fd1ee41fb1b992702521ae078eb',
'f2d9790f2752a794517b949c65a773eb864844cd',
])
}
}
module.exports = new TestGitlabContextParser();

19
components/server/src/gitlab/gitlab-context-parser.ts
View File

@ -394,23 +394,4 @@ export class GitlabContextParser extends AbstractContextParser implements IConte
};
}
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
// TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled.
const { owner, repoName } = await this.parseURL(user, contextUrl);
const projectId = `${owner}/${repoName}`;
const result = await this.gitlabApi.run<GitLab.Commit[]>(user, async g => {
return g.Commits.all(projectId, {
ref_name: sha,
per_page: maxDepth,
page: 1,
});
});
if (GitLab.ApiError.is(result)) {
if (result.message === 'GitLab responded with code 404') {
throw new Error(`Couldn't find commit #${sha} in repository ${projectId}.`);
}
throw result;
}
return result.slice(1).map((c: GitLab.Commit) => c.id);
}
}

64
components/server/src/gitlab/gitlab-repository-provider.spec.ts Normal file
View File

@ -0,0 +1,64 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
import { User } from "@gitpod/gitpod-protocol";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { expect } from "chai";
import { Container, ContainerModule } from "inversify";
import { suite, retries, test, timeout } from "mocha-typescript";
import { AuthProviderParams } from "../auth/auth-provider";
import { HostContextProvider } from "../auth/host-context-provider";
import { DevData } from "../dev/dev-data";
import { TokenProvider } from "../user/token-provider";
import { GitLabApi } from "./api";
import { GitlabContextParser } from "./gitlab-context-parser";
import { GitlabRepositoryProvider } from "./gitlab-repository-provider";
import { GitLabTokenHelper } from "./gitlab-token-helper";
@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITLAB"))
class TestGitlabRepositoryProvider {
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitLab",
type: "GitLab",
verified: true,
description: "",
icon: "",
host: "gitlab.com",
}
protected repositoryProvider: GitlabRepositoryProvider;
protected user: User;
public before() {
const container = new Container();
container.load(new ContainerModule((bind, unbind, isBound, rebind) => {
bind(GitlabContextParser).toSelf().inSingletonScope();
bind(GitLabApi).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestGitlabRepositoryProvider.AUTH_HOST_CONFIG);
bind(GitLabTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async () => DevData.createGitlabTestToken(),
getFreshPortAuthenticationToken: async (user: User, workspaceId: string) => DevData.createPortAuthTestToken(workspaceId),
});
bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider());
bind(GitlabRepositoryProvider).toSelf().inSingletonScope();
}));
this.repositoryProvider = container.get(GitlabRepositoryProvider);
this.user = DevData.createTestUser();
}
@test public async testFetchCommitHistory() {
const result = await this.repositoryProvider.getCommitHistory(this.user, 'AlexTugarev', 'gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100);
expect(result).to.deep.equal([
'4447fbc4d46e6fd1ee41fb1b992702521ae078eb',
'f2d9790f2752a794517b949c65a773eb864844cd',
])
}
}
module.exports = new TestGitlabRepositoryProvider();

19
components/server/src/gitlab/gitlab-repository-provider.ts
View File

@ -113,4 +113,23 @@ export class GitlabRepositoryProvider implements RepositoryProvider {
return false;
}
}
public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise<string[]> {
// TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled.
const projectId = `${owner}/${repo}`;
const result = await this.gitlab.run<GitLab.Commit[]>(user, async g => {
return g.Commits.all(projectId, {
ref_name: ref,
per_page: maxDepth,
page: 1,
});
});
if (GitLab.ApiError.is(result)) {
if (result.message === 'GitLab responded with code 404') {
throw new Error(`Couldn't find commit #${ref} in repository ${projectId}.`);
}
throw result;
}
return result.slice(1).map((c: GitLab.Commit) => c.id);
}
}

4
components/server/src/gitlab/gitlab-token-helper.ts
View File

@ -31,8 +31,8 @@ export class GitLabTokenHelper {
if (this.containsScopes(token, requiredScopes)) {
return token;
}
} catch {
// no token
} catch (e) {
console.error(e);
}
if (requiredScopes.length === 0) {
requiredScopes = GitLabScope.Requirements.DEFAULT

4
components/server/src/projects/projects-service.ts
View File

@ -94,9 +94,7 @@ export class ProjectsService {
changeHash: commit.sha,
changeTitle: commit.commitMessage,
changeAuthorAvatar: commit.authorAvatarUrl,
isDefault: repository.defaultBranch === branch.name,
changePR: "changePR", // todo: compute in repositoryProvider
changeUrl: "changeUrl", // todo: compute in repositoryProvider
isDefault: repository.defaultBranch === branch.name
});
}
result.sort((a, b) => (b.changeDate || "").localeCompare(a.changeDate || ""));

1
components/server/src/repohost/repository-provider.ts
View File

@ -15,4 +15,5 @@ export interface RepositoryProvider {
getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined>;
getUserRepos(user: User): Promise<string[]>;
hasReadAccess(user: User, owner: string, repo: string): Promise<boolean>;
getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise<string[]>;
}

109
components/server/src/workspace/context-parser-service.ts
View File

@ -65,21 +65,25 @@ export class ContextParser {
protected async internalHandleWithoutPrefix(ctx: TraceContext, user: User, nonPrefixedContextURL: string): Promise<WorkspaceContext> {
const span = TraceContext.startSpan("ContextParser.internalHandle", ctx);
let result: WorkspaceContext | undefined;
try {
let result: WorkspaceContext | undefined;
for (const parser of this.allContextParsers) {
if (parser.canHandle(user, nonPrefixedContextURL)) {
result = await parser.handle({ span }, user, nonPrefixedContextURL);
break;
for (const parser of this.allContextParsers) {
if (parser.canHandle(user, nonPrefixedContextURL)) {
result = await parser.handle({ span }, user, nonPrefixedContextURL);
break;
}
}
if (!result) {
throw new Error(`Couldn't parse context '${nonPrefixedContextURL}'.`);
}
}
if (!result) {
throw new Error(`Couldn't parse context '${nonPrefixedContextURL}'.`);
}
// TODO: Make the parsers return the context with normalizedContextURL set
result.normalizedContextURL = nonPrefixedContextURL;
return result;
// TODO: Make the parsers return the context with normalizedContextURL set
result.normalizedContextURL = nonPrefixedContextURL;
return result;
} finally {
span.finish();
}
}
protected buildUpstreamCloneUrl(context: CommitContext): string | undefined {
@ -101,47 +105,52 @@ export class ContextParser {
return context;
}
const span = TraceContext.startSpan("ContextParser.handleMultiRepositoryContext", ctx);
let config = await this.configProvider.fetchConfig({ span }, user, context);
let mainRepoContext: WorkspaceContext | undefined;
if (config.config.mainRepository) {
mainRepoContext = await this.internalHandleWithoutPrefix({ span }, user, config.config.mainRepository);
if (!CommitContext.is(mainRepoContext)) {
throw new InvalidGitpodYMLError([`Cannot find main repository '${config.config.mainRepository}'.`]);
}
config = await this.configProvider.fetchConfig({ span }, user, mainRepoContext);
}
if (config.config.subRepositories && config.config.subRepositories.length > 0) {
const subRepoCommits: GitCheckoutInfo[] = [];
for (const subRepo of config.config.subRepositories) {
let subContext = await this.internalHandleWithoutPrefix({ span }, user, subRepo.url) as CommitContext;
if (!CommitContext.is(subContext)) {
throw new InvalidGitpodYMLError([`Cannot find sub-repository '${subRepo.url}'.`]);
try {
let config = await this.configProvider.fetchConfig({ span }, user, context);
let mainRepoContext: WorkspaceContext | undefined;
if (config.config.mainConfiguration) {
mainRepoContext = await this.internalHandleWithoutPrefix({ span }, user, config.config.mainConfiguration);
if (!CommitContext.is(mainRepoContext)) {
throw new InvalidGitpodYMLError([`Cannot find main repository '${config.config.mainConfiguration}'.`]);
}
if (context.repository.cloneUrl === subContext.repository.cloneUrl) {
// if it's the repo from the original context we want to use that commit.
subContext = JSON.parse(JSON.stringify(context));
}
subRepoCommits.push({
... subContext,
checkoutLocation: (subRepo.checkoutLocation || subContext.repository.name),
upstreamRemoteURI: this.buildUpstreamCloneUrl(subContext),
localBranch: context.localBranch // we want to create a local branch on all repos, in case it's a multi-repo change. If it's not there are no drawbacks anyway.
});
config = await this.configProvider.fetchConfig({ span }, user, mainRepoContext);
}
context.subRepositoryCheckoutInfo = subRepoCommits;
if (config.config.additionalRepositories && config.config.additionalRepositories.length > 0) {
const subRepoCommits: GitCheckoutInfo[] = [];
for (const subRepo of config.config.additionalRepositories) {
let subContext = await this.internalHandleWithoutPrefix({ span }, user, subRepo.url) as CommitContext;
if (!CommitContext.is(subContext)) {
throw new InvalidGitpodYMLError([`Cannot find sub-repository '${subRepo.url}'.`]);
}
if (context.repository.cloneUrl === subContext.repository.cloneUrl) {
// if it's the repo from the original context we want to use that commit.
subContext = JSON.parse(JSON.stringify(context));
}
subRepoCommits.push({
... subContext,
checkoutLocation: (subRepo.checkoutLocation || subContext.repository.name),
upstreamRemoteURI: this.buildUpstreamCloneUrl(subContext),
// we want to create a local branch on all repos, in case it's a multi-repo change. If it's not there are no drawbacks anyway.
ref: context.ref,
refType: context.refType,
localBranch: context.localBranch
});
}
context.additionalRepositoryCheckoutInfo = subRepoCommits;
}
// if the original contexturl was pointing to a subrepo we update the commit information with the mainContext.
if (mainRepoContext && CommitContext.is(mainRepoContext)) {
context.repository = mainRepoContext.repository;
context.revision = mainRepoContext.revision;
}
context.checkoutLocation = (config.config.checkoutLocation || context.repository.name);
context.upstreamRemoteURI = this.buildUpstreamCloneUrl(context);
return context;
} finally {
span.finish();
}
// if the original contexturl was pointing to a subrepo we update the commit information with the mainContext.
if (mainRepoContext && CommitContext.is(mainRepoContext)) {
context.repository = mainRepoContext.repository;
context.revision = mainRepoContext.revision;
context.ref = mainRepoContext.revision;
context.refType = mainRepoContext.refType;
}
context.checkoutLocation = (config.config.checkoutLocation || context.repository.name);
context.upstreamRemoteURI = this.buildUpstreamCloneUrl(context);
return context;
}
protected findPrefix(user: User, context: string): { prefix: string, parser: IPrefixContextParser } | undefined {

8
components/server/src/workspace/context-parser.ts
View File

@ -14,7 +14,6 @@ export interface IContextParser {
normalize?(contextUrl: string): string | undefined
canHandle(user: User, contextUrl: string): boolean
handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>
fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise<string[] | undefined>
}
export const IContextParser = Symbol("IContextParser")
@ -78,13 +77,6 @@ export abstract class AbstractContextParser implements IContextParser {
}
public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>;
/**
* Fetches the commit history of a commit (used to find a relevant parent prebuild for incremental prebuilds).
*
* @returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log`
*/
public abstract fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise<string[] | undefined>;
}
export interface URLParts {

18
components/server/src/workspace/workspace-starter.ts
View File

@ -7,7 +7,7 @@
import { CloneTargetMode, FileDownloadInitializer, GitAuthMethod, GitConfig, GitInitializer, PrebuildInitializer, SnapshotInitializer, WorkspaceInitializer } from "@gitpod/content-service/lib";
import { CompositeInitializer, FromBackupInitializer } from "@gitpod/content-service/lib/initializer_pb";
import { DBUser, DBWithTracing, ProjectDB, TracedUserDB, TracedWorkspaceDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib';
import { CommitContext, Disposable, GitpodToken, GitpodTokenType, IssueContext, NamedWorkspaceFeatureFlag, PullRequestContext, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ImageBuildLogInfo, ProjectEnvVar } from "@gitpod/gitpod-protocol";
import { CommitContext, Disposable, GitpodToken, GitpodTokenType, GitCheckoutInfo, NamedWorkspaceFeatureFlag, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ProjectEnvVar, ImageBuildLogInfo } from "@gitpod/gitpod-protocol";
import { IAnalyticsWriter } from '@gitpod/gitpod-protocol/lib/analytics';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
@ -1016,9 +1016,13 @@ export class WorkspaceStarter {
const init = new PrebuildInitializer();
init.setPrebuild(snapshot);
if (initializer instanceof CompositeInitializer) {
init.setComposite(initializer);
for (const myInit of initializer.getInitializerList()) {
if (myInit instanceof WorkspaceInitializer && myInit.hasGit()) {
init.addGit(myInit.getGit());
}
}
} else {
init.setGit(initializer);
init.addGit(initializer);
}
result.setPrebuild(init);
} else if (WorkspaceProbeContext.is(context)) {
@ -1073,13 +1077,13 @@ export class WorkspaceStarter {
protected async createCommitInitializer(ctx: TraceContext, workspace: Workspace, context: CommitContext, user: User): Promise<{initializer: GitInitializer | CompositeInitializer, disposable: Disposable}> {
const span = TraceContext.startSpan("createInitializerForCommit", ctx);
const mainGit = this.createGitInitializer({ span }, workspace, context, user);
if (!context.subRepositoryCheckoutInfo || context.subRepositoryCheckoutInfo.length === 0) {
if (!context.additionalRepositoryCheckoutInfo || context.additionalRepositoryCheckoutInfo.length === 0) {
return mainGit;
}
const subRepoInitializers = [mainGit];
await Promise.all(context.subRepositoryCheckoutInfo.map(async subRepo => {
subRepoInitializers.push(this.createGitInitializer({ span }, workspace, subRepo , user));
}));
for (const subRepo of context.additionalRepositoryCheckoutInfo) {
subRepoInitializers.push(this.createGitInitializer({ span }, workspace, subRepo , user));
}
const inits = await Promise.all(subRepoInitializers);
const compositeInit = new CompositeInitializer();
const compositeDisposable = new DisposableCollection();

6
components/ws-daemon/pkg/content/service.go
View File

@ -282,8 +282,10 @@ func getCheckoutLocation(req *api.InitWorkspaceRequest) string {
}
}
if ir, ok := spec.(*csapi.WorkspaceInitializer_Prebuild); ok {
if ir.Prebuild != nil && ir.Prebuild.Git != nil {
return ir.Prebuild.Git.CheckoutLocation
if ir.Prebuild != nil {
if len(ir.Prebuild.Git) > 0 {
return ir.Prebuild.Git[0].CheckoutLocation
}
}
}
return ""

3
scripts/mysql.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
kubectl port-forward statefulset/mysql 3306:3306 &
mysql -h 127.0.0.1 -P 3306 -u gitpod -D gitpod --select-limit=200 --safe-updates --password="$(kubectl get secrets mysql -o jsonpath="{.data.password}" | base64 -d)"