archive-gh-me/gitpod
1
0
Fork 0
mirror of https://github.com/gitpod-io/gitpod.git synced 2025-12-08 17:36:30 +00:00
Code Issues Projects Releases Wiki Activity

[multi-repo] Support prebuilds on multi-repo setup

Browse Source
This commit is contained in:
Sven Efftinge 2022-03-09 17:42:50 +00:00 committed by Robo Quat
parent 2e1774eaf7
commit cc2f3b30fe
70 changed files with 943 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
components/content-service-api/go/blobs_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved. // Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL). // Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information. // See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/content_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved. // Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL). // Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information. // See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/headless-log_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved. // Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL). // Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information. // See License-AGPL.txt in the project root for license information.

2
components/content-service-api/go/ideplugin_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved. // Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL). // Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information. // See License-AGPL.txt in the project root for license information.

110
components/content-service-api/go/initializer.pb.go
View File

@ -662,9 +662,8 @@ type PrebuildInitializer struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
Prebuild *SnapshotInitializer `protobuf:"bytes,1,opt,name=prebuild,proto3" json:"prebuild,omitempty"` Prebuild *SnapshotInitializer `protobuf:"bytes,1,opt,name=prebuild,proto3" json:"prebuild,omitempty"`
Git *GitInitializer `protobuf:"bytes,2,opt,name=git,proto3" json:"git,omitempty"` Git []*GitInitializer `protobuf:"bytes,2,rep,name=git,proto3" json:"git,omitempty"`
Composite *CompositeInitializer `protobuf:"bytes,3,opt,name=composite,proto3" json:"composite,omitempty"`
} }
func (x *PrebuildInitializer) Reset() { func (x *PrebuildInitializer) Reset() {
@ -706,20 +705,13 @@ func (x *PrebuildInitializer) GetPrebuild() *SnapshotInitializer {
return nil return nil
} }
func (x *PrebuildInitializer) GetGit() *GitInitializer { func (x *PrebuildInitializer) GetGit() []*GitInitializer {
if x != nil { if x != nil {
return x.Git return x.Git
} }
return nil return nil
} }
func (x *PrebuildInitializer) GetComposite() *CompositeInitializer {
if x != nil {
return x.Composite
}
return nil
}
// FromBackupInitializer initializes content from a previously made backup // FromBackupInitializer initializes content from a previously made backup
type FromBackupInitializer struct { type FromBackupInitializer struct {
state protoimpl.MessageState state protoimpl.MessageState
@ -1039,58 +1031,53 @@ var file_initializer_proto_rawDesc = []byte{
0x3a, 0x02, 0x38, 0x01, 0x22, 0x31, 0x0a, 0x13, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x31, 0x0a, 0x13, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x22, 0xcc, 0x01, 0x0a, 0x13, 0x50, 0x72, 0x65, 0x62, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x22, 0x88, 0x01, 0x0a, 0x13, 0x50, 0x72, 0x65, 0x62,
0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x12,
0x3f, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x3f, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69,
0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64,
0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e,
0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47,
0x69, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x03, 0x67, 0x69, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x03, 0x67,
0x69, 0x74, 0x12, 0x42, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x18, 0x69, 0x74, 0x22, 0x17, 0x0a, 0x15, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70,
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x22, 0xe7, 0x02, 0x0a, 0x09,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x47, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x72, 0x61,
0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6e, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x72, 0x61, 0x6e, 0x63,
0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d,
0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x22, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74,
0xe7, 0x02, 0x0a, 0x09, 0x47, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d,
0x06, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x52, 0x0f, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65,
0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x61, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x63, 0x6f, 0x6d,
0x74, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x75, 0x6e, 0x74, 0x72, 0x61,
0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09,
0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x52, 0x0e, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73,
0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x6f, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63,
0x6d, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x75, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52,
0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x75, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64,
0x69, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f,
0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12,
0x01, 0x28, 0x03, 0x52, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65,
0x6b, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x6e, 0x70, 0x75, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52,
0x73, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f,
0x28, 0x09, 0x52, 0x0f, 0x75, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x2a, 0x5a, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x61,
0x69, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x70, 0x72, 0x67, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x4d, 0x4f,
0x75, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x08, 0x20, 0x54, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x4d,
0x01, 0x28, 0x03, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x70, 0x75, 0x73, 0x68, 0x4f, 0x54, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d,
0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x2a, 0x5a, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43, 0x48, 0x10, 0x02, 0x12,
0x6e, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x10, 0x0a, 0x0c, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43, 0x48, 0x10,
0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x03, 0x2a, 0x40, 0x0a, 0x0d, 0x47, 0x69, 0x74, 0x41, 0x75, 0x74, 0x68, 0x4d, 0x65, 0x74, 0x68,
0x0d, 0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01, 0x6f, 0x64, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x10, 0x00, 0x12,
0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x42, 0x52, 0x41, 0x4e, 0x43, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x10, 0x01, 0x12,
0x48, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x42, 0x52, 0x41, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x5f, 0x4f, 0x54,
0x4e, 0x43, 0x48, 0x10, 0x03, 0x2a, 0x40, 0x0a, 0x0d, 0x47, 0x69, 0x74, 0x41, 0x75, 0x74, 0x68, 0x53, 0x10, 0x02, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x5f, 0x41, 0x55, 0x54, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2d, 0x69, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x70,
0x48, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x6f, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x69,
0x48, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x53, 0x49, 0x43, 0x5f, 0x41, 0x55, 0x54, 0x63, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x48, 0x5f, 0x4f, 0x54, 0x53, 0x10, 0x02, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2d, 0x69, 0x6f, 0x2f,
0x67, 0x69, 0x74, 0x70, 0x6f, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
} }
var ( var (
@ -1139,12 +1126,11 @@ var file_initializer_proto_depIdxs = []int32{
1, // 12: contentservice.GitConfig.authentication:type_name -> contentservice.GitAuthMethod 1, // 12: contentservice.GitConfig.authentication:type_name -> contentservice.GitAuthMethod
8, // 13: contentservice.PrebuildInitializer.prebuild:type_name -> contentservice.SnapshotInitializer 8, // 13: contentservice.PrebuildInitializer.prebuild:type_name -> contentservice.SnapshotInitializer
6, // 14: contentservice.PrebuildInitializer.git:type_name -> contentservice.GitInitializer 6, // 14: contentservice.PrebuildInitializer.git:type_name -> contentservice.GitInitializer
3, // 15: contentservice.PrebuildInitializer.composite:type_name -> contentservice.CompositeInitializer 15, // [15:15] is the sub-list for method output_type
16, // [16:16] is the sub-list for method output_type 15, // [15:15] is the sub-list for method input_type
16, // [16:16] is the sub-list for method input_type 15, // [15:15] is the sub-list for extension type_name
16, // [16:16] is the sub-list for extension type_name 15, // [15:15] is the sub-list for extension extendee
16, // [16:16] is the sub-list for extension extendee 0, // [0:15] is the sub-list for field type_name
0, // [0:16] is the sub-list for field type_name
} }
func init() { file_initializer_proto_init() } func init() { file_initializer_proto_init() }

2
components/content-service-api/go/workspace_grpc.pb.go
View File

@ -1,4 +1,4 @@
// Copyright (c) 2021 Gitpod GmbH. All rights reserved. // Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL). // Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information. // See License-AGPL.txt in the project root for license information.

3
components/content-service-api/initializer.proto
View File

@ -118,8 +118,7 @@ message SnapshotInitializer {
// If restoring the snapshot fails, we fall back to a regular Git initializer, which might be composite git initializer for multi-repo projects. // If restoring the snapshot fails, we fall back to a regular Git initializer, which might be composite git initializer for multi-repo projects.
message PrebuildInitializer { message PrebuildInitializer {
SnapshotInitializer prebuild = 1; SnapshotInitializer prebuild = 1;
GitInitializer git = 2; repeated GitInitializer git = 2;
CompositeInitializer composite = 3;
} }
// FromBackupInitializer initializes content from a previously made backup // FromBackupInitializer initializes content from a previously made backup

2
components/content-service-api/typescript/src/blobs_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/blobs_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/blobs_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/content_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/content_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/content_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/headless-log_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/headless-log_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/ideplugin_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/ideplugin_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/initializer_grpc_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

17
components/content-service-api/typescript/src/initializer_pb.d.ts vendored
View File

@ -275,16 +275,10 @@ export class PrebuildInitializer extends jspb.Message {
clearPrebuild(): void; clearPrebuild(): void;
getPrebuild(): SnapshotInitializer | undefined; getPrebuild(): SnapshotInitializer | undefined;
setPrebuild(value?: SnapshotInitializer): PrebuildInitializer; setPrebuild(value?: SnapshotInitializer): PrebuildInitializer;
clearGitList(): void;
hasGit(): boolean; getGitList(): Array<GitInitializer>;
clearGit(): void; setGitList(value: Array<GitInitializer>): PrebuildInitializer;
getGit(): GitInitializer | undefined; addGit(value?: GitInitializer, index?: number): GitInitializer;
setGit(value?: GitInitializer): PrebuildInitializer;
hasComposite(): boolean;
clearComposite(): void;
getComposite(): CompositeInitializer | undefined;
setComposite(value?: CompositeInitializer): PrebuildInitializer;
serializeBinary(): Uint8Array; serializeBinary(): Uint8Array;
toObject(includeInstance?: boolean): PrebuildInitializer.AsObject; toObject(includeInstance?: boolean): PrebuildInitializer.AsObject;
@ -299,8 +293,7 @@ export class PrebuildInitializer extends jspb.Message {
export namespace PrebuildInitializer { export namespace PrebuildInitializer {
export type AsObject = { export type AsObject = {
prebuild?: SnapshotInitializer.AsObject, prebuild?: SnapshotInitializer.AsObject,
git?: GitInitializer.AsObject, gitList: Array<GitInitializer.AsObject>,
composite?: CompositeInitializer.AsObject,
} }
} }

112
components/content-service-api/typescript/src/initializer_pb.js
View File

@ -220,7 +220,7 @@ if (goog.DEBUG && !COMPILED) {
* @constructor * @constructor
*/ */
proto.contentservice.PrebuildInitializer = function(opt_data) { proto.contentservice.PrebuildInitializer = function(opt_data) {
jspb.Message.initialize(this, opt_data, 0, -1, null, null); jspb.Message.initialize(this, opt_data, 0, -1, proto.contentservice.PrebuildInitializer.repeatedFields_, null);
}; };
goog.inherits(proto.contentservice.PrebuildInitializer, jspb.Message); goog.inherits(proto.contentservice.PrebuildInitializer, jspb.Message);
if (goog.DEBUG && !COMPILED) { if (goog.DEBUG && !COMPILED) {
@ -2086,6 +2086,13 @@ proto.contentservice.SnapshotInitializer.prototype.setSnapshot = function(value)
/**
* List of repeated fields within this message type.
* @private {!Array<number>}
* @const
*/
proto.contentservice.PrebuildInitializer.repeatedFields_ = [2];
if (jspb.Message.GENERATE_TO_OBJECT) { if (jspb.Message.GENERATE_TO_OBJECT) {
@ -2118,8 +2125,8 @@ proto.contentservice.PrebuildInitializer.prototype.toObject = function(opt_inclu
proto.contentservice.PrebuildInitializer.toObject = function(includeInstance, msg) { proto.contentservice.PrebuildInitializer.toObject = function(includeInstance, msg) {
var f, obj = { var f, obj = {
prebuild: (f = msg.getPrebuild()) && proto.contentservice.SnapshotInitializer.toObject(includeInstance, f), prebuild: (f = msg.getPrebuild()) && proto.contentservice.SnapshotInitializer.toObject(includeInstance, f),
git: (f = msg.getGit()) && proto.contentservice.GitInitializer.toObject(includeInstance, f), gitList: jspb.Message.toObjectList(msg.getGitList(),
composite: (f = msg.getComposite()) && proto.contentservice.CompositeInitializer.toObject(includeInstance, f) proto.contentservice.GitInitializer.toObject, includeInstance)
}; };
if (includeInstance) { if (includeInstance) {
@ -2164,12 +2171,7 @@ proto.contentservice.PrebuildInitializer.deserializeBinaryFromReader = function(
case 2: case 2:
var value = new proto.contentservice.GitInitializer; var value = new proto.contentservice.GitInitializer;
reader.readMessage(value,proto.contentservice.GitInitializer.deserializeBinaryFromReader); reader.readMessage(value,proto.contentservice.GitInitializer.deserializeBinaryFromReader);
msg.setGit(value); msg.addGit(value);
break;
case 3:
var value = new proto.contentservice.CompositeInitializer;
reader.readMessage(value,proto.contentservice.CompositeInitializer.deserializeBinaryFromReader);
msg.setComposite(value);
break; break;
default: default:
reader.skipField(); reader.skipField();
@ -2208,22 +2210,14 @@ proto.contentservice.PrebuildInitializer.serializeBinaryToWriter = function(mess
proto.contentservice.SnapshotInitializer.serializeBinaryToWriter proto.contentservice.SnapshotInitializer.serializeBinaryToWriter
); );
} }
f = message.getGit(); f = message.getGitList();
if (f != null) { if (f.length > 0) {
writer.writeMessage( writer.writeRepeatedMessage(
2, 2,
f, f,
proto.contentservice.GitInitializer.serializeBinaryToWriter proto.contentservice.GitInitializer.serializeBinaryToWriter
); );
} }
f = message.getComposite();
if (f != null) {
writer.writeMessage(
3,
f,
proto.contentservice.CompositeInitializer.serializeBinaryToWriter
);
}
}; };
@ -2265,76 +2259,40 @@ proto.contentservice.PrebuildInitializer.prototype.hasPrebuild = function() {
/** /**
* optional GitInitializer git = 2; * repeated GitInitializer git = 2;
* @return {?proto.contentservice.GitInitializer} * @return {!Array<!proto.contentservice.GitInitializer>}
*/ */
proto.contentservice.PrebuildInitializer.prototype.getGit = function() { proto.contentservice.PrebuildInitializer.prototype.getGitList = function() {
return /** @type{?proto.contentservice.GitInitializer} */ ( return /** @type{!Array<!proto.contentservice.GitInitializer>} */ (
jspb.Message.getWrapperField(this, proto.contentservice.GitInitializer, 2)); jspb.Message.getRepeatedWrapperField(this, proto.contentservice.GitInitializer, 2));
}; };
/** /**
* @param {?proto.contentservice.GitInitializer|undefined} value * @param {!Array<!proto.contentservice.GitInitializer>} value
* @return {!proto.contentservice.PrebuildInitializer} returns this * @return {!proto.contentservice.PrebuildInitializer} returns this
*/ */
proto.contentservice.PrebuildInitializer.prototype.setGit = function(value) { proto.contentservice.PrebuildInitializer.prototype.setGitList = function(value) {
return jspb.Message.setWrapperField(this, 2, value); return jspb.Message.setRepeatedWrapperField(this, 2, value);
}; };
/** /**
* Clears the message field making it undefined. * @param {!proto.contentservice.GitInitializer=} opt_value
* @param {number=} opt_index
* @return {!proto.contentservice.GitInitializer}
*/
proto.contentservice.PrebuildInitializer.prototype.addGit = function(opt_value, opt_index) {
return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.contentservice.GitInitializer, opt_index);
};
/**
* Clears the list making it empty but non-null.
* @return {!proto.contentservice.PrebuildInitializer} returns this * @return {!proto.contentservice.PrebuildInitializer} returns this
*/ */
proto.contentservice.PrebuildInitializer.prototype.clearGit = function() { proto.contentservice.PrebuildInitializer.prototype.clearGitList = function() {
return this.setGit(undefined); return this.setGitList([]);
};
/**
* Returns whether this field is set.
* @return {boolean}
*/
proto.contentservice.PrebuildInitializer.prototype.hasGit = function() {
return jspb.Message.getField(this, 2) != null;
};
/**
* optional CompositeInitializer composite = 3;
* @return {?proto.contentservice.CompositeInitializer}
*/
proto.contentservice.PrebuildInitializer.prototype.getComposite = function() {
return /** @type{?proto.contentservice.CompositeInitializer} */ (
jspb.Message.getWrapperField(this, proto.contentservice.CompositeInitializer, 3));
};
/**
* @param {?proto.contentservice.CompositeInitializer|undefined} value
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.setComposite = function(value) {
return jspb.Message.setWrapperField(this, 3, value);
};
/**
* Clears the message field making it undefined.
* @return {!proto.contentservice.PrebuildInitializer} returns this
*/
proto.contentservice.PrebuildInitializer.prototype.clearComposite = function() {
return this.setComposite(undefined);
};
/**
* Returns whether this field is set.
* @return {boolean}
*/
proto.contentservice.PrebuildInitializer.prototype.hasComposite = function() {
return jspb.Message.getField(this, 3) != null;
}; };

2
components/content-service-api/typescript/src/workspace_grpc_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/workspace_pb.d.ts vendored
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

2
components/content-service-api/typescript/src/workspace_pb.js
View File

@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */

3
components/content-service/pkg/initializer/git.go
View File

@ -141,7 +141,8 @@ func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) {
if ws.TargetMode == RemoteBranch { if ws.TargetMode == RemoteBranch {
// create local branch based on specific remote branch // create local branch based on specific remote branch
if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/"+ws.CloneTarget); err != nil { if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/"+ws.CloneTarget); err != nil {
return err log.WithField("remoteURI", ws.RemoteURI).WithField("branch", ws.CloneTarget).Debug("Remote branch doesn't exist.")
return nil
} }
} else if ws.TargetMode == LocalBranch { } else if ws.TargetMode == LocalBranch {
// checkout local branch based on remote HEAD // checkout local branch based on remote HEAD

19
components/content-service/pkg/initializer/initializer.go
View File

@ -111,14 +111,6 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader
if ir.Prebuild == nil { if ir.Prebuild == nil {
return nil, status.Error(codes.InvalidArgument, "missing prebuild initializer spec") return nil, status.Error(codes.InvalidArgument, "missing prebuild initializer spec")
} }
if ir.Prebuild.Git == nil {
return nil, status.Error(codes.InvalidArgument, "missing prebuild Git initializer spec")
}
gitinit, err := newGitInitializer(ctx, loc, ir.Prebuild.Git, opts.ForceGitpodUserForGit)
if err != nil {
return nil, err
}
var snapshot *SnapshotInitializer var snapshot *SnapshotInitializer
if ir.Prebuild.Prebuild != nil { if ir.Prebuild.Prebuild != nil {
snapshot, err = newSnapshotInitializer(loc, rs, ir.Prebuild.Prebuild) snapshot, err = newSnapshotInitializer(loc, rs, ir.Prebuild.Prebuild)
@ -126,10 +118,17 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader
return nil, status.Error(codes.Internal, fmt.Sprintf("cannot setup prebuild init: %v", err)) return nil, status.Error(codes.Internal, fmt.Sprintf("cannot setup prebuild init: %v", err))
} }
} }
var gits []*GitInitializer
for _, gi := range ir.Prebuild.Git {
gitinit, err := newGitInitializer(ctx, loc, gi, opts.ForceGitpodUserForGit)
if err != nil {
return nil, err
}
gits = append(gits, gitinit)
}
initializer = &PrebuildInitializer{ initializer = &PrebuildInitializer{
Git: gitinit,
Prebuild: snapshot, Prebuild: snapshot,
Git: gits,
} }
} else if ir, ok := spec.(*csapi.WorkspaceInitializer_Snapshot); ok { } else if ir, ok := spec.(*csapi.WorkspaceInitializer_Snapshot); ok {
initializer, err = newSnapshotInitializer(loc, rs, ir.Snapshot) initializer, err = newSnapshotInitializer(loc, rs, ir.Snapshot)

100
components/content-service/pkg/initializer/prebuild.go
View File

@ -26,7 +26,7 @@ import (
// PrebuildInitializer first tries to restore the snapshot/prebuild and if that succeeds performs Git operations. // PrebuildInitializer first tries to restore the snapshot/prebuild and if that succeeds performs Git operations.
// If restoring the prebuild does not succeed we fall back to Git entriely. // If restoring the prebuild does not succeed we fall back to Git entriely.
type PrebuildInitializer struct { type PrebuildInitializer struct {
Git *GitInitializer Git []*GitInitializer
Prebuild *SnapshotInitializer Prebuild *SnapshotInitializer
} }
@ -45,14 +45,11 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
tracelog.String("snapshot", p.Prebuild.Snapshot), tracelog.String("snapshot", p.Prebuild.Snapshot),
) )
} }
if p.Git == nil { if len(p.Git) == 0 {
spandata = append(spandata, tracelog.Bool("hasGit", false)) spandata = append(spandata, tracelog.Bool("hasGit", false))
} else { } else {
spandata = append(spandata, spandata = append(spandata,
tracelog.Bool("hasGit", true), tracelog.Bool("hasGit", true),
tracelog.String("git.targetMode", string(p.Git.TargetMode)),
tracelog.String("git.cloneTarget", string(p.Git.CloneTarget)),
tracelog.String("git.location", string(p.Git.Location)),
) )
} }
span.LogFields(spandata...) span.LogFields(spandata...)
@ -71,7 +68,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: %w", err) return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: %w", err)
} }
return p.Git.Run(ctx, mappings) for _, gi := range p.Git {
_, err = gi.Run(ctx, mappings)
if err != nil {
return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: Git fallback: %w", err)
}
}
} }
} }
@ -80,47 +82,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
src = csapi.WorkspaceInitFromPrebuild src = csapi.WorkspaceInitFromPrebuild
// make sure we're on the correct branch // make sure we're on the correct branch
span.LogFields( for _, gi := range p.Git {
tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(p.Git.Location))), err = runGitInit(ctx, gi)
tracelog.String("location", fmt.Sprintf("%v", p.Git.Location)),
)
if git.IsWorkingCopy(p.Git.Location) {
out, err := p.Git.GitWithOutput(ctx, "stash", "push", "-u")
if err != nil { if err != nil {
var giterr git.OpFailedError return src, err
if errors.As(err, &giterr) && strings.Contains(giterr.Output, "You do not have the initial commit yet") {
// git stash push returns a non-zero exit code if the repository does not have a single commit.
// In this case that's not an error though, hence we don't want to fail here.
} else {
// git returned a non-zero exit code because of some reason we did not anticipate or an actual failure.
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
} }
didStash := !strings.Contains(string(out), "No local changes to save")
err = p.Git.Fetch(ctx)
if err != nil {
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
err = p.Git.realizeCloneTarget(ctx)
if err != nil {
return src, xerrors.Errorf("prebuild initializer: %w", err)
}
// If any of these cleanup operations fail that's no reason to fail ws initialization.
// It just results in a slightly degraded state.
if didStash {
err = p.Git.Git(ctx, "stash", "pop")
if err != nil {
// If restoring the stashed changes produces merge conflicts on the new Git ref, simply
// throw them away (they'll remain in the stash, but are likely outdated anyway).
_ = p.Git.Git(ctx, "reset", "--hard")
}
}
log.Debug("prebuild initializer Git operations complete")
} }
log.Debug("Initialized workspace with prebuilt snapshot") log.Debug("Initialized workspace with prebuilt snapshot")
return return
} }
@ -138,3 +105,48 @@ func clearWorkspace(location string) error {
} }
return nil return nil
} }
func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) {
span, ctx := opentracing.StartSpanFromContext(ctx, "runGitInit")
span.LogFields(
tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(gInit.Location))),
tracelog.String("location", fmt.Sprintf("%v", gInit.Location)),
)
if git.IsWorkingCopy(gInit.Location) {
out, err := gInit.GitWithOutput(ctx, "stash", "push", "-u")
if err != nil {
var giterr git.OpFailedError
if errors.As(err, &giterr) && strings.Contains(giterr.Output, "You do not have the initial commit yet") {
// git stash push returns a non-zero exit code if the repository does not have a single commit.
// In this case that's not an error though, hence we don't want to fail here.
} else {
// git returned a non-zero exit code because of some reason we did not anticipate or an actual failure.
return xerrors.Errorf("prebuild initializer: %w", err)
}
}
didStash := !strings.Contains(string(out), "No local changes to save")
err = gInit.Fetch(ctx)
if err != nil {
return xerrors.Errorf("prebuild initializer: %w", err)
}
err = gInit.realizeCloneTarget(ctx)
if err != nil {
return xerrors.Errorf("prebuild initializer: %w", err)
}
// If any of these cleanup operations fail that's no reason to fail ws initialization.
// It just results in a slightly degraded state.
if didStash {
err = gInit.Git(ctx, "stash", "pop")
if err != nil {
// If restoring the stashed changes produces merge conflicts on the new Git ref, simply
// throw them away (they'll remain in the stash, but are likely outdated anyway).
_ = gInit.Git(ctx, "reset", "--hard")
}
}
log.Debug("prebuild initializer Git operations complete")
}
return nil
}

16
components/content-service/pkg/layer/provider.go
View File

@ -184,7 +184,21 @@ func (s *Provider) GetContentLayer(ctx context.Context, owner, workspaceID strin
span.LogKV("fallback-to-git", err.Error()) span.LogKV("fallback-to-git", err.Error())
// we failed creating a prebuild initializer, so let's try falling back to the Git part. // we failed creating a prebuild initializer, so let's try falling back to the Git part.
initializer = &csapi.WorkspaceInitializer{Spec: &csapi.WorkspaceInitializer_Git{Git: pis.Git}} var init []*csapi.WorkspaceInitializer
for _, gi := range pis.Git {
init = append(init, &csapi.WorkspaceInitializer{
Spec: &csapi.WorkspaceInitializer_Git{
Git: gi,
},
})
}
initializer = &csapi.WorkspaceInitializer{
Spec: &csapi.WorkspaceInitializer_Composite{
Composite: &csapi.CompositeInitializer{
Initializer: init,
},
},
}
} else { } else {
// creating the initializer worked - we're done here // creating the initializer worked - we're done here
return return

12
components/dashboard/src/projects/Prebuilds.tsx
View File

@ -219,14 +219,18 @@ export default function (props: { project?: Project, isAdminDashboard?: boolean
</ItemField> </ItemField>
<ItemField className="flex items-center my-auto"> <ItemField className="flex items-center my-auto">
<div className="truncate"> <div className="truncate">
<div className="text-base text-gray-500 dark:text-gray-50 font-medium mb-1 truncate" title={shortCommitMessage(p.info.changeTitle)}>{shortCommitMessage(p.info.changeTitle)}</div> <a href={p.info.changeUrl} className="cursor-pointer">
<div className="text-base text-gray-500 dark:text-gray-50 font-medium mb-1 truncate" title={shortCommitMessage(p.info.changeTitle)}>{shortCommitMessage(p.info.changeTitle)}</div>
</a>
<p>{p.info.changeAuthorAvatar && <img className="rounded-full w-4 h-4 inline-block align-text-bottom mr-2" src={p.info.changeAuthorAvatar || ''} alt={p.info.changeAuthor} />}Authored {formatDate(p.info.changeDate)} · {p.info.changeHash?.substring(0, 8)}</p> <p>{p.info.changeAuthorAvatar && <img className="rounded-full w-4 h-4 inline-block align-text-bottom mr-2" src={p.info.changeAuthorAvatar || ''} alt={p.info.changeAuthor} />}Authored {formatDate(p.info.changeDate)} · {p.info.changeHash?.substring(0, 8)}</p>
</div> </div>
</ItemField> </ItemField>
<ItemField className="flex"> <ItemField className="flex">
<div className="flex space-x-2 truncate"> <a href={p.info.changeUrl} className="cursor-pointer">
<span className="font-medium text-gray-500 dark:text-gray-50 truncate" title={p.info.branch}>{p.info.branch}</span> <div className="flex space-x-2 truncate">
</div> <span className="font-medium text-gray-500 dark:text-gray-50 truncate" title={p.info.branch}>{p.info.branch}</span>
</div>
</a>
<span className="flex-grow" /> <span className="flex-grow" />
{!props.isAdminDashboard && <ItemFieldContextMenu menuEntries={prebuildContextMenu(p)} />} {!props.isAdminDashboard && <ItemFieldContextMenu menuEntries={prebuildContextMenu(p)} />}
</ItemField> </ItemField>

6
components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts
View File

@ -31,6 +31,12 @@ export class DBPrebuiltWorkspaceUpdatable implements PrebuiltWorkspaceUpdatable
@Column() @Column()
repo: string; repo: string;
@Column({
default: '',
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED
})
commitSHA?: string;
@Column() @Column()
isResolved: boolean; isResolved: boolean;

21
components/gitpod-db/src/typeorm/migration/1646803519382-PrebuildUpdatableSHA.ts Normal file
View File

@ -0,0 +1,21 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
import {MigrationInterface, QueryRunner} from "typeorm";
import { columnExists } from "./helper/helper";
export class PrebuildUpdatableSHA1646803519382 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
if (!(await columnExists(queryRunner, "d_b_prebuilt_workspace_updatable", "commitSHA"))) {
await queryRunner.query("ALTER TABLE d_b_prebuilt_workspace_updatable ADD COLUMN commitSHA varchar(255) NOT NULL");
}
}
public async down(queryRunner: QueryRunner): Promise<void> {
}
}

3
components/gitpod-db/src/typeorm/workspace-db-impl.ts
View File

@ -633,8 +633,9 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
let query = repo.createQueryBuilder('pws'); let query = repo.createQueryBuilder('pws');
query = query.where('pws.cloneURL = :cloneURL', { cloneURL }) query = query.where('pws.cloneURL = :cloneURL', { cloneURL })
query = query.orderBy('pws.creationTime', 'ASC'); query = query.orderBy('pws.creationTime', 'DESC');
query = query.innerJoinAndMapOne('pws.workspace', DBWorkspace, 'ws', 'pws.buildWorkspaceId = ws.id'); query = query.innerJoinAndMapOne('pws.workspace', DBWorkspace, 'ws', 'pws.buildWorkspaceId = ws.id');
query = query.where('ws.deleted = false');
const res = await query.getMany(); const res = await query.getMany();
return res.map(r => { return res.map(r => {

6
components/gitpod-protocol/data/gitpod-schema.json
View File

@ -136,9 +136,9 @@
}, },
"additionalProperties": false "additionalProperties": false
}, },
"subRepositories": { "additionalRepositories": {
"type": "array", "type": "array",
"description": "List of sub repositories that are part of this project.", "description": "List of additional repositories that are part of this project.",
"items": { "items": {
"type": "object", "type": "object",
"required": [ "required": [
@ -157,7 +157,7 @@
"additionalProperties": false "additionalProperties": false
} }
}, },
"mainRepository": { "mainConfiguration": {
"type": "string", "type": "string",
"description": "The main repository, containing the dev environment configuration." "description": "The main repository, containing the dev environment configuration."
}, },

42
components/gitpod-protocol/src/protocol.ts
View File

@ -7,6 +7,7 @@
import { WorkspaceInstance, PortVisibility } from "./workspace-instance"; import { WorkspaceInstance, PortVisibility } from "./workspace-instance";
import { RoleOrPermission } from "./permission"; import { RoleOrPermission } from "./permission";
import { Project } from "./teams-projects-protocol"; import { Project } from "./teams-projects-protocol";
import { createHash } from "crypto";
export interface UserInfo { export interface UserInfo {
name?: string name?: string
@ -567,14 +568,14 @@ export interface VSCodeConfig {
extensions?: string[]; extensions?: string[];
} }
export interface SubRepository { export interface RepositoryCloneInformation {
url: string; url: string;
checkoutLocation?: string; checkoutLocation?: string;
} }
export interface WorkspaceConfig { export interface WorkspaceConfig {
mainRepository?: string; mainConfiguration?: string;
subRepositories?: SubRepository[]; additionalRepositories?: RepositoryCloneInformation[];
image?: ImageConfig; image?: ImageConfig;
ports?: PortConfig[]; ports?: PortConfig[];
tasks?: TaskConfig[]; tasks?: TaskConfig[];
@ -695,6 +696,10 @@ export interface PrebuiltWorkspaceUpdatable {
repo: string; repo: string;
isResolved: boolean; isResolved: boolean;
installationId: string; installationId: string;
/**
* the commitSHA of the commit that triggered the prebuild
*/
commitSHA?: string;
issue?: string; issue?: string;
contextUrl?: string; contextUrl?: string;
} }
@ -875,6 +880,10 @@ export namespace SnapshotContext {
export interface StartPrebuildContext extends WorkspaceContext { export interface StartPrebuildContext extends WorkspaceContext {
actual: WorkspaceContext; actual: WorkspaceContext;
commitHistory?: string[]; commitHistory?: string[];
additionalRepositoryCommitHistories?: {
cloneUrl: string;
commitHistory: string[];
}[];
project?: Project; project?: Project;
branch?: string; branch?: string;
} }
@ -982,9 +991,32 @@ export interface CommitContext extends WorkspaceContext, GitCheckoutInfo {
cloneUrl?: string cloneUrl?: string
/** /**
* The clone and checkout information for the sub-repositories in case of multi-repo projects. * The clone and checkout information for additional repositories in case of multi-repo projects.
*/ */
subRepositoryCheckoutInfo?: GitCheckoutInfo[]; additionalRepositoryCheckoutInfo?: GitCheckoutInfo[];
}
export namespace CommitContext {
/**
* Creates a hash for all the commits of the CommitContext and all sub-repo commit infos.
* The hash is max 255 chars long.
* @param commitContext
* @returns hash for commitcontext
*/
export function computeHash(commitContext: CommitContext): string {
// for single commits we use the revision to be backward compatible.
if (!commitContext.additionalRepositoryCheckoutInfo || commitContext.additionalRepositoryCheckoutInfo.length === 0) {
return commitContext.revision;
}
const hasher = createHash('sha256');
hasher.update(commitContext.revision);
for (const info of commitContext.additionalRepositoryCheckoutInfo) {
hasher.update(info.revision);
}
return hasher.digest('hex');
}
} }
export interface GitCheckoutInfo extends Commit { export interface GitCheckoutInfo extends Commit {

4
components/gitpod-protocol/src/wsready.ts
View File

@ -1,10 +1,10 @@
/** /**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved. * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL). * Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */
// generated using github.com/32leaves/bel on 2021-11-04 12:16:53.917570766 +0000 UTC m=+0.006002884 // generated using github.com/32leaves/bel on 2022-02-15 11:53:18.380158212 +0000 UTC m=+0.011913675
// DO NOT MODIFY // DO NOT MODIFY
export enum WorkspaceInitSource { export enum WorkspaceInitSource {

2
components/server/ee/src/container-module.ts
View File

@ -26,7 +26,6 @@ import { BitbucketApp } from "./prebuilds/bitbucket-app";
import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app"; import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app";
import { IPrefixContextParser } from "../../src/workspace/context-parser"; import { IPrefixContextParser } from "../../src/workspace/context-parser";
import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser"; import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser";
import { StartIncrementalPrebuildContextParser } from "./prebuilds/start-incremental-prebuild-context-parser";
import { WorkspaceFactory } from "../../src/workspace/workspace-factory"; import { WorkspaceFactory } from "../../src/workspace/workspace-factory";
import { WorkspaceFactoryEE } from "./workspace/workspace-factory"; import { WorkspaceFactoryEE } from "./workspace/workspace-factory";
import { MonitoringEndpointsAppEE } from "./monitoring-endpoint-ee"; import { MonitoringEndpointsAppEE } from "./monitoring-endpoint-ee";
@ -60,7 +59,6 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
bind(WorkspaceHealthMonitoring).toSelf().inSingletonScope(); bind(WorkspaceHealthMonitoring).toSelf().inSingletonScope();
bind(PrebuildManager).toSelf().inSingletonScope(); bind(PrebuildManager).toSelf().inSingletonScope();
bind(IPrefixContextParser).to(StartPrebuildContextParser).inSingletonScope(); bind(IPrefixContextParser).to(StartPrebuildContextParser).inSingletonScope();
bind(IPrefixContextParser).to(StartIncrementalPrebuildContextParser).inSingletonScope();
bind(GithubApp).toSelf().inSingletonScope(); bind(GithubApp).toSelf().inSingletonScope();
bind(GitHubAppSupport).toSelf().inSingletonScope(); bind(GitHubAppSupport).toSelf().inSingletonScope();
bind(GithubAppRules).toSelf().inSingletonScope(); bind(GithubAppRules).toSelf().inSingletonScope();

30
components/server/ee/src/prebuilds/bitbucket-app.ts
View File

@ -7,10 +7,13 @@
import * as express from 'express'; import * as express from 'express';
import { postConstruct, injectable, inject } from 'inversify'; import { postConstruct, injectable, inject } from 'inversify';
import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib'; import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib';
import { User, StartPrebuildResult, Project } from '@gitpod/gitpod-protocol'; import { User, StartPrebuildResult, CommitContext, CommitInfo, Project } from '@gitpod/gitpod-protocol';
import { PrebuildManager } from '../prebuilds/prebuild-manager'; import { PrebuildManager } from '../prebuilds/prebuild-manager';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { TokenService } from '../../../src/user/token-service'; import { TokenService } from '../../../src/user/token-service';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
@injectable() @injectable()
export class BitbucketApp { export class BitbucketApp {
@ -20,6 +23,8 @@ export class BitbucketApp {
@inject(TokenService) protected readonly tokenService: TokenService; @inject(TokenService) protected readonly tokenService: TokenService;
@inject(ProjectDB) protected readonly projectDB: ProjectDB; @inject(ProjectDB) protected readonly projectDB: ProjectDB;
@inject(TeamDB) protected readonly teamDB: TeamDB; @inject(TeamDB) protected readonly teamDB: TeamDB;
@inject(ContextParser) protected readonly contextParser: ContextParser;
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
protected _router = express.Router(); protected _router = express.Router();
public static path = '/apps/bitbucket/'; public static path = '/apps/bitbucket/';
@ -88,25 +93,24 @@ export class BitbucketApp {
const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx); const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx);
try { try {
const contextURL = this.createContextUrl(data); const contextURL = this.createContextUrl(data);
const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
span.setTag('contextURL', contextURL); span.setTag('contextURL', contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); const config = await this.prebuildManager.fetchConfig({ span }, user, context);
if (!this.prebuildManager.shouldPrebuild(config)) { if (!this.prebuildManager.shouldPrebuild(config)) {
console.log('Bitbucket push event: No config. No prebuild.'); console.log('Bitbucket push event: No config. No prebuild.');
return undefined; return undefined;
} }
console.debug('Bitbucket push event: Starting prebuild.', { contextURL }); console.log('Starting prebuild.', { contextURL })
const {host, owner, repo} = RepoURL.parseRepoUrl(data.repoUrl)!;
const hostCtx = this.hostCtxProvider.get(host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx.services.repositoryProvider.getCommitInfo(user, owner, repo, data.commitHash);
}
const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user); const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user);
// todo@alex: add branch and project args
const ws = await this.prebuildManager.startPrebuild({ span }, { const ws = await this.prebuildManager.startPrebuild({ span }, { user, project: projectAndOwner?.project, context, commitInfo });
user: projectAndOwner.user,
project: projectAndOwner?.project,
branch: data.branchName,
contextURL,
cloneURL: data.gitCloneUrl,
commit: data.commitHash
});
return ws; return ws;
} finally { } finally {
span.finish(); span.finish();

107
components/server/ee/src/prebuilds/github-app.ts
View File

@ -12,13 +12,17 @@ import { Config } from '../../../src/config';
import { AppInstallationDB, TracedWorkspaceDB, DBWithTracing, UserDB, WorkspaceDB, ProjectDB, TeamDB } from '@gitpod/gitpod-db/lib'; import { AppInstallationDB, TracedWorkspaceDB, DBWithTracing, UserDB, WorkspaceDB, ProjectDB, TeamDB } from '@gitpod/gitpod-db/lib';
import * as express from 'express'; import * as express from 'express';
import { log, LogContext, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging'; import { log, LogContext, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
import { WorkspaceConfig, User, Project, StartPrebuildResult } from '@gitpod/gitpod-protocol'; import { WorkspaceConfig, User, Project, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol';
import { GithubAppRules } from './github-app-rules'; import { GithubAppRules } from './github-app-rules';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { PrebuildManager } from './prebuild-manager'; import { PrebuildManager } from './prebuild-manager';
import { PrebuildStatusMaintainer } from './prebuilt-status-maintainer'; import { PrebuildStatusMaintainer } from './prebuilt-status-maintainer';
import { Options, ApplicationFunctionOptions } from 'probot/lib/types'; import { Options, ApplicationFunctionOptions } from 'probot/lib/types';
import { asyncHandler } from '../../../src/express-util'; import { asyncHandler } from '../../../src/express-util';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
/** /**
* GitHub app urls: * GitHub app urls:
@ -39,6 +43,8 @@ export class GithubApp {
@inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing<WorkspaceDB>; @inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing<WorkspaceDB>;
@inject(GithubAppRules) protected readonly appRules: GithubAppRules; @inject(GithubAppRules) protected readonly appRules: GithubAppRules;
@inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager;
@inject(ContextParser) protected readonly contextParser: ContextParser;
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
readonly server: Server | undefined; readonly server: Server | undefined;
@ -159,6 +165,19 @@ export class GithubApp {
}); });
} }
private async findOwnerAndProject(installationID: number | undefined, cloneURL: string): Promise<{ user: User, project?: Project }> {
const installationOwner = installationID ? await this.findInstallationOwner(installationID) : undefined;
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: cloneURL, installationID, project });
throw new Error(`No installation found for ${installationID}`);
}
return {
user, project
}
}
protected async handlePushEvent(ctx: Context<'push'>): Promise<void> { protected async handlePushEvent(ctx: Context<'push'>): Promise<void> {
const span = TraceContext.startSpan("GithubApp.handlePushEvent", {}); const span = TraceContext.startSpan("GithubApp.handlePushEvent", {});
span.setTag("request", ctx.id); span.setTag("request", ctx.id);
@ -166,13 +185,7 @@ export class GithubApp {
try { try {
const installationId = ctx.payload.installation?.id; const installationId = ctx.payload.installation?.id;
const cloneURL = ctx.payload.repository.clone_url; const cloneURL = ctx.payload.repository.clone_url;
const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined; let { user, project } = await this.findOwnerAndProject(installationId, cloneURL);
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: ctx.payload.repository, installationId, project });
return;
}
const logCtx: LogContext = { userId: user.id }; const logCtx: LogContext = { userId: user.id };
if (!!user.blocked) { if (!!user.blocked) {
@ -192,7 +205,13 @@ export class GithubApp {
const contextURL = `${repo.html_url}/tree/${branch}`; const contextURL = `${repo.html_url}/tree/${branch}`;
span.setTag('contextURL', contextURL); span.setTag('contextURL', contextURL);
let config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
const r = await this.ensureMainProjectAndUser(user, project, context, installationId);
user = r.user;
project = r.project;
const runPrebuild = this.appRules.shouldRunPrebuild(config, branch == repo.default_branch, false, false); const runPrebuild = this.appRules.shouldRunPrebuild(config, branch == repo.default_branch, false, false);
if (!runPrebuild) { if (!runPrebuild) {
const reason = `Not running prebuild, the user did not enable it for this context`; const reason = `Not running prebuild, the user did not enable it for this context`;
@ -201,7 +220,8 @@ export class GithubApp {
return; return;
} }
this.prebuildManager.startPrebuild({ span }, { user, contextURL, cloneURL: repo.clone_url, commit: pl.after, branch, project}) const commitInfo = await this.getCommitInfo(user, repo.html_url, ctx.payload.after);
this.prebuildManager.startPrebuild({ span }, { user, context, project, commitInfo})
.catch(err => log.error(logCtx, "Error while starting prebuild", err, { contextURL })); .catch(err => log.error(logCtx, "Error while starting prebuild", err, { contextURL }));
} catch (e) { } catch (e) {
TraceContext.setError({ span }, e); TraceContext.setError({ span }, e);
@ -211,6 +231,33 @@ export class GithubApp {
} }
} }
private async ensureMainProjectAndUser(user: User, project: Project | undefined, context: CommitContext, installationId?: number): Promise<{user: User, project?: Project}> {
// if it's a sub-repo of a multi-repo project, we look up the owner of the main repo
if (!!context.additionalRepositoryCheckoutInfo && (!project || context.repository.cloneUrl !== project.cloneUrl)) {
const owner = await this.findOwnerAndProject(installationId, context.repository.cloneUrl);
if (owner) {
return {
user: owner.user,
project: owner.project || project
};
}
}
return {
user,
project
};
}
private async getCommitInfo(user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = this.hostCtxProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
protected getBranchFromRef(ref: string): string | undefined { protected getBranchFromRef(ref: string): string | undefined {
const headsPrefix = "refs/heads/"; const headsPrefix = "refs/heads/";
if (ref.startsWith(headsPrefix)) { if (ref.startsWith(headsPrefix)) {
@ -227,22 +274,23 @@ export class GithubApp {
try { try {
const installationId = ctx.payload.installation?.id; const installationId = ctx.payload.installation?.id;
const cloneURL = ctx.payload.repository.clone_url; const cloneURL = ctx.payload.repository.clone_url;
const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined;
const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
const user = await this.selectUserForPrebuild(installationOwner, project);
if (!user) {
log.info("Did not find user for installation. Probably an incomplete app installation.", { repo: ctx.payload.repository, installationId, project });
return;
}
const pr = ctx.payload.pull_request; const pr = ctx.payload.pull_request;
const contextURL = pr.html_url; const contextURL = pr.html_url;
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); let { user, project} = await this.findOwnerAndProject(installationId, cloneURL);
const prebuildStartPromise = this.onPrStartPrebuild({ span }, ctx, config, user, project); const context = await this.contextParser.handle( { span }, user, contextURL) as CommitContext;
this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise).catch(() => {/** ignore */}); const config = await this.prebuildManager.fetchConfig({ span }, user, context);
this.onPrAddBadge(config, ctx);
this.onPrAddComment(config, ctx).catch(() => {/** ignore */}); const r = await this.ensureMainProjectAndUser(user, project, context, installationId);
user = r.user;
project = r.project;
const prebuildStartPromise = await this.onPrStartPrebuild({ span }, ctx, config, context, user, project);
if (prebuildStartPromise) {
await this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise);
this.onPrAddBadge(config, ctx);
await this.onPrAddComment(config, ctx);
}
} catch (e) { } catch (e) {
TraceContext.setError({ span }, e); TraceContext.setError({ span }, e);
throw e; throw e;
@ -251,7 +299,7 @@ export class GithubApp {
} }
} }
protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: Promise<StartPrebuildResult> | undefined) { protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: StartPrebuildResult) {
if (!start) { if (!start) {
return; return;
} }
@ -262,8 +310,7 @@ export class GithubApp {
const span = TraceContext.startSpan("onPrAddCheck", tracecContext); const span = TraceContext.startSpan("onPrAddCheck", tracecContext);
try { try {
const spr = await start; const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(start.wsid);
const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(spr.wsid);
if (!pws) { if (!pws) {
return; return;
} }
@ -286,18 +333,16 @@ export class GithubApp {
} }
} }
protected onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig | undefined, user: User, project?: Project): Promise<StartPrebuildResult> | undefined { protected async onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig, context: CommitContext, user: User, project?: Project): Promise<StartPrebuildResult | undefined> {
const pr = ctx.payload.pull_request; const pr = ctx.payload.pull_request;
const pr_head = pr.head;
const contextURL = pr.html_url; const contextURL = pr.html_url;
const branch = pr.head.ref;
const cloneURL = pr_head.repo.clone_url;
const isFork = pr.head.repo.id !== pr.base.repo.id; const isFork = pr.head.repo.id !== pr.base.repo.id;
const runPrebuild = this.appRules.shouldRunPrebuild(config, false, true, isFork); const runPrebuild = this.appRules.shouldRunPrebuild(config, false, true, isFork);
let prebuildStartPromise: Promise<StartPrebuildResult> | undefined; let prebuildStartPromise: Promise<StartPrebuildResult> | undefined;
if (runPrebuild) { if (runPrebuild) {
prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, contextURL, cloneURL, commit: pr_head.sha, branch, project}); const commitInfo = await this.getCommitInfo(user, ctx.payload.repository.html_url, pr.head.sha);
prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, context, project, commitInfo});
prebuildStartPromise.catch(err => log.error(err, "Error while starting prebuild", { contextURL })); prebuildStartPromise.catch(err => log.error(err, "Error while starting prebuild", { contextURL }));
return prebuildStartPromise; return prebuildStartPromise;
} else { } else {

35
components/server/ee/src/prebuilds/gitlab-app.ts
View File

@ -7,13 +7,15 @@
import * as express from 'express'; import * as express from 'express';
import { postConstruct, injectable, inject } from 'inversify'; import { postConstruct, injectable, inject } from 'inversify';
import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib'; import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib';
import { Project, User, StartPrebuildResult } from '@gitpod/gitpod-protocol'; import { Project, User, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol';
import { PrebuildManager } from '../prebuilds/prebuild-manager'; import { PrebuildManager } from '../prebuilds/prebuild-manager';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { TokenService } from '../../../src/user/token-service'; import { TokenService } from '../../../src/user/token-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider'; import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { GitlabService } from './gitlab-service'; import { GitlabService } from './gitlab-service';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { ContextParser } from '../../../src/workspace/context-parser-service';
import { RepoURL } from '../../../src/repohost';
@injectable() @injectable()
export class GitLabApp { export class GitLabApp {
@ -24,6 +26,7 @@ export class GitLabApp {
@inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider;
@inject(ProjectDB) protected readonly projectDB: ProjectDB; @inject(ProjectDB) protected readonly projectDB: ProjectDB;
@inject(TeamDB) protected readonly teamDB: TeamDB; @inject(TeamDB) protected readonly teamDB: TeamDB;
@inject(ContextParser) protected readonly contextParser: ContextParser;
protected _router = express.Router(); protected _router = express.Router();
public static path = '/apps/gitlab/'; public static path = '/apps/gitlab/';
@ -96,7 +99,9 @@ export class GitLabApp {
const contextURL = this.createContextUrl(body); const contextURL = this.createContextUrl(body);
log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL }); log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL });
span.setTag('contextURL', contextURL); span.setTag('contextURL', contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext;
const projectAndOwner = await this.findProjectAndOwner(context.repository.cloneUrl, user);
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
if (!this.prebuildManager.shouldPrebuild(config)) { if (!this.prebuildManager.shouldPrebuild(config)) {
log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { context: body, contextURL }); log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { context: body, contextURL });
return undefined; return undefined;
@ -104,18 +109,12 @@ export class GitLabApp {
log.debug({ userId: user.id }, "GitLab push hook: Starting prebuild", { body, contextURL }); log.debug({ userId: user.id }, "GitLab push hook: Starting prebuild", { body, contextURL });
const cloneURL = body.repository.git_http_url; const commitInfo = await this.getCommitInfo(user, body.repository.git_http_url, body.after);
const branch = this.getBranchFromRef(body.ref);
const projectAndOwner = await this.findProjectAndOwner(cloneURL, user);
const ws = await this.prebuildManager.startPrebuild({ span }, { const ws = await this.prebuildManager.startPrebuild({ span }, {
user: projectAndOwner.user, user: projectAndOwner?.user || user,
project: projectAndOwner?.project, project: projectAndOwner?.project,
contextURL, context,
cloneURL, commitInfo
commit: body.after,
branch,
}); });
return ws; return ws;
@ -124,6 +123,16 @@ export class GitLabApp {
} }
} }
private async getCommitInfo(user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = this.hostCtxProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
/** /**
* Finds the relevant user account and project to the provided webhook event information. * Finds the relevant user account and project to the provided webhook event information.
* *
@ -135,7 +144,7 @@ export class GitLabApp {
* @param webhookInstaller the user account known from the webhook installation * @param webhookInstaller the user account known from the webhook installation
* @returns a promise which resolves to a user account and an optional project. * @returns a promise which resolves to a user account and an optional project.
*/ */
protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> { protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> {
const project = await this.projectDB.findProjectByCloneUrl(cloneURL); const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
if (project) { if (project) {
if (project.userId) { if (project.userId) {

152
components/server/ee/src/prebuilds/prebuild-manager.ts
View File

@ -5,10 +5,10 @@
*/ */
import { DBWithTracing, TracedWorkspaceDB, WorkspaceDB } from '@gitpod/gitpod-db/lib'; import { DBWithTracing, TracedWorkspaceDB, WorkspaceDB } from '@gitpod/gitpod-db/lib';
import { CommitContext, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol'; import { CommitContext, CommitInfo, PrebuiltWorkspace, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, Workspace, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
import { HostContextProvider } from '../../../src/auth/host-context-provider'; import { getCommitInfo, HostContextProvider } from '../../../src/auth/host-context-provider';
import { WorkspaceFactory } from '../../../src/workspace/workspace-factory'; import { WorkspaceFactory } from '../../../src/workspace/workspace-factory';
import { ConfigProvider } from '../../../src/workspace/config-provider'; import { ConfigProvider } from '../../../src/workspace/config-provider';
import { WorkspaceStarter } from '../../../src/workspace/workspace-starter'; import { WorkspaceStarter } from '../../../src/workspace/workspace-starter';
@ -18,7 +18,6 @@ import { secondsBefore } from '@gitpod/gitpod-protocol/lib/util/timeutil';
import { inject, injectable } from 'inversify'; import { inject, injectable } from 'inversify';
import * as opentracing from 'opentracing'; import * as opentracing from 'opentracing';
import { URL } from 'url';
export class WorkspaceRunningError extends Error { export class WorkspaceRunningError extends Error {
constructor(msg: string, public instance: WorkspaceInstance) { constructor(msg: string, public instance: WorkspaceInstance) {
@ -28,11 +27,9 @@ export class WorkspaceRunningError extends Error {
export interface StartPrebuildParams { export interface StartPrebuildParams {
user: User; user: User;
contextURL: string; context: CommitContext;
cloneURL: string;
branch?: string;
commit: string;
project?: Project; project?: Project;
commitInfo?: CommitInfo;
} }
const PREBUILD_LIMITER_WINDOW_SECONDS = 60; const PREBUILD_LIMITER_WINDOW_SECONDS = 60;
@ -48,43 +45,25 @@ export class PrebuildManager {
@inject(Config) protected readonly config: Config; @inject(Config) protected readonly config: Config;
@inject(ProjectsService) protected readonly projectService: ProjectsService; @inject(ProjectsService) protected readonly projectService: ProjectsService;
async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise<boolean> { async startPrebuild(ctx: TraceContext, { context, project, user, commitInfo }: StartPrebuildParams): Promise<StartPrebuildResult> {
const span = TraceContext.startSpan("hasPrebuilds", ctx);
span.setTag(cloneURL, cloneURL);
try {
const existingPBs = await this.workspaceDB.trace({ span }).findPrebuildsWithWorkpace(cloneURL);
for (const pb of existingPBs) {
if (!pb.workspace.contextURL.startsWith('prebuild')) {
return true;
}
}
return false;
} catch (err) {
TraceContext.setError({ span }, err);
throw err;
} finally {
span.finish();
}
}
async startPrebuild(ctx: TraceContext, { contextURL, cloneURL, commit, branch, project, user }: StartPrebuildParams): Promise<StartPrebuildResult> {
const span = TraceContext.startSpan("startPrebuild", ctx); const span = TraceContext.startSpan("startPrebuild", ctx);
span.setTag("contextURL", contextURL); const cloneURL = context.repository.cloneUrl;
const commitSHAIdentifier = CommitContext.computeHash(context);
span.setTag("cloneURL", cloneURL); span.setTag("cloneURL", cloneURL);
span.setTag("commit", commit); span.setTag("commit", commitInfo?.sha);
try { try {
if (user.blocked) { if (user.blocked) {
throw new Error("Blocked users cannot start prebuilds."); throw new Error("Blocked users cannot start prebuilds.");
} }
const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commit); const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commitSHAIdentifier);
// If the existing prebuild is failed, we want to retrigger it. // If the existing prebuild is failed, we want to retrigger it.
if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'failed' && existingPB.state !== 'timeout') { if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'failed' && existingPB.state !== 'timeout') {
// If the existing prebuild is based on an outdated project config, we also want to retrigger it. // If the existing prebuild is based on an outdated project config, we also want to retrigger it.
const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId); const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId);
const existingConfig = existingPBWS?.config; const existingConfig = existingPBWS?.config;
const newConfig = await this.fetchConfig({ span }, user, contextURL); const newConfig = await this.fetchConfig({ span }, user, context);
log.debug(`startPrebuild | commit: ${commit}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`); log.debug(`startPrebuild | commits: ${commitSHAIdentifier}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`);
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task) .map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key)) .filter(key => ['before', 'init', 'prebuild'].includes(key))
@ -98,35 +77,46 @@ export class PrebuildManager {
} }
} }
const contextParser = this.getContextParserFor(contextURL);
if (!contextParser) {
throw new Error(`Cannot find context parser for URL: ${contextURL}`);
}
const actual = await contextParser.handle({ span }, user, contextURL) as CommitContext;
actual.revision = commit; // Make sure we target the correct commit here (might have changed between trigger and contextParser lookup)
actual.ref = undefined;
actual.forceCreateNewWorkspace = true;
const prebuildContext: StartPrebuildContext = { const prebuildContext: StartPrebuildContext = {
title: `Prebuild of "${actual.title}"`, title: `Prebuild of "${context.title}"`,
actual, actual: context,
project, project,
branch, branch: context.ref,
normalizedContextURL: actual.normalizedContextURL normalizedContextURL: context.normalizedContextURL
}; };
if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl, project)) { if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) {
const maxDepth = this.config.incrementalPrebuilds.commitHistory; const maxDepth = this.config.incrementalPrebuilds.commitHistory;
prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth); const hostContext = this.hostContextProvider.get(context.repository.host);
const repoProvider = hostContext?.services?.repositoryProvider;
if (repoProvider) {
prebuildContext.commitHistory = await repoProvider.getCommitHistory(user, context.repository.owner, context.repository.name, context.revision, maxDepth);
if (context.additionalRepositoryCheckoutInfo && context.additionalRepositoryCheckoutInfo.length > 0) {
const histories = context.additionalRepositoryCheckoutInfo.map(async info => {
const commitHistory = await repoProvider.getCommitHistory(user, info.repository.owner, info.repository.name, info.revision, maxDepth);
return {
cloneUrl: info.repository.cloneUrl,
commitHistory
}
});
prebuildContext.additionalRepositoryCommitHistories = await Promise.all(histories);
}
}
} }
const projectEnvVarsPromise = project ? this.projectService.getProjectEnvironmentVariables(project.id) : []; const projectEnvVarsPromise = project ? this.projectService.getProjectEnvironmentVariables(project.id) : [];
const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL);
const prebuild = await this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!; const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, context.normalizedContextURL!);
const prebuildPromise = this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!;
span.setTag("starting", true);
const projectEnvVars = await projectEnvVarsPromise;
await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']});
const prebuild = await prebuildPromise;
if (!prebuild) { if (!prebuild) {
throw new Error(`Failed to create a prebuild for: ${contextURL}`); throw new Error(`Failed to create a prebuild for: ${context.normalizedContextURL}`);
} }
if (await this.shouldRateLimitPrebuild(span, cloneURL)) { if (await this.shouldRateLimitPrebuild(span, cloneURL)) {
@ -143,9 +133,20 @@ export class PrebuildManager {
}; };
} }
span.setTag("starting", true); if (project) {
const projectEnvVars = await projectEnvVarsPromise; let aCommitInfo = commitInfo;
await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']}); if (!aCommitInfo) {
aCommitInfo = await getCommitInfo(this.hostContextProvider, user, context.repository.cloneUrl, context.revision);
if (!aCommitInfo) {
aCommitInfo = {
author: 'unknown',
commitMessage: 'unknown',
sha: context.revision
}
}
}
await this.storePrebuildInfo({ span }, project, prebuild, workspace, user, aCommitInfo);
}
return { prebuildId: prebuild.id, wsid: workspace.id, done: false }; return { prebuildId: prebuild.id, wsid: workspace.id, done: false };
} catch (err) { } catch (err) {
TraceContext.setError({ span }, err); TraceContext.setError({ span }, err);
@ -214,17 +215,10 @@ export class PrebuildManager {
return this.config.incrementalPrebuilds.repositoryPasslist.some(url => trimRepoUrl(url) === repoUrl); return this.config.incrementalPrebuilds.repositoryPasslist.some(url => trimRepoUrl(url) === repoUrl);
} }
async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise<WorkspaceConfig | undefined> { async fetchConfig(ctx: TraceContext, user: User, context: CommitContext): Promise<WorkspaceConfig> {
const span = TraceContext.startSpan("fetchConfig", ctx); const span = TraceContext.startSpan("fetchConfig", ctx);
span.setTag("contextURL", contextURL);
try { try {
const contextParser = this.getContextParserFor(contextURL); return (await this.configProvider.fetchConfig({ span }, user, context)).config;
if (!contextParser) {
return undefined;
}
const context = await contextParser!.handle({ span }, user, contextURL);
return (await this.configProvider.fetchConfig({ span }, user, context as CommitContext)).config;
} catch (err) { } catch (err) {
TraceContext.setError({ span }, err); TraceContext.setError({ span }, err);
throw err; throw err;
@ -233,13 +227,31 @@ export class PrebuildManager {
} }
} }
protected getContextParserFor(contextURL: string) { //TODO this doesn't belong so deep here. All this context should be stored on the surface not passed down.
const host = new URL(contextURL).hostname; protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User, commit: CommitInfo) {
const hostContext = this.hostContextProvider.get(host); const span = TraceContext.startSpan("storePrebuildInfo", ctx);
if (!hostContext) { const { userId, teamId, name: projectName, id: projectId } = project;
return undefined; await this.workspaceDB.trace({span}).storePrebuildInfo({
} id: pws.id,
return hostContext.contextParser; buildWorkspaceId: pws.buildWorkspaceId,
basedOnPrebuildId: ws.basedOnPrebuildId,
teamId,
userId,
projectName,
projectId,
startedAt: pws.creationTime,
startedBy: "", // TODO
startedByAvatar: "", // TODO
cloneUrl: pws.cloneURL,
branch: pws.branch || "unknown",
changeAuthor: commit.author,
changeAuthorAvatar: commit.authorAvatarUrl,
changeDate: commit.authorDate || "",
changeHash: commit.sha,
changeTitle: commit.commitMessage,
// changePR
changeUrl: ws.contextURL,
});
} }
private async shouldRateLimitPrebuild(span: opentracing.Span, cloneURL: string): Promise<boolean> { private async shouldRateLimitPrebuild(span: opentracing.Span, cloneURL: string): Promise<boolean> {
@ -270,4 +282,4 @@ export class PrebuildManager {
// Last resort default // Last resort default
return PREBUILD_LIMITER_DEFAULT_LIMIT; return PREBUILD_LIMITER_DEFAULT_LIMIT;
} }
} }

35
components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts
View File

@ -46,7 +46,7 @@ export class PrebuildStatusMaintainer implements Disposable {
this.disposables.push( this.disposables.push(
repeat(this.periodicUpdatableCheck.bind(this), 60 * 1000) repeat(this.periodicUpdatableCheck.bind(this), 60 * 1000)
); );
log.debug("prebuild updatatable status maintainer started"); log.debug("prebuild updatable status maintainer started");
} }
public async registerCheckRun(ctx: TraceContext, installationId: number, pws: PrebuiltWorkspace, cri: CheckRunInfo, config?: WorkspaceConfig) { public async registerCheckRun(ctx: TraceContext, installationId: number, pws: PrebuiltWorkspace, cri: CheckRunInfo, config?: WorkspaceConfig) {
@ -64,6 +64,7 @@ export class PrebuildStatusMaintainer implements Disposable {
id: uuidv4(), id: uuidv4(),
owner: cri.owner, owner: cri.owner,
repo: cri.repo, repo: cri.repo,
commitSHA: cri.head_sha,
isResolved: false, isResolved: false,
installationId: installationId.toString(), installationId: installationId.toString(),
contextUrl: cri.details_url, contextUrl: cri.details_url,
@ -131,8 +132,8 @@ export class PrebuildStatusMaintainer implements Disposable {
return; return;
} }
const updatatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id); const updatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id);
await Promise.all(updatatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild))); await Promise.all(updatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild)));
} catch (err) { } catch (err) {
TraceContext.setError({span}, err); TraceContext.setError({span}, err);
throw err; throw err;
@ -141,38 +142,38 @@ export class PrebuildStatusMaintainer implements Disposable {
} }
} }
protected async doUpdate(ctx: TraceContext, updatatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise<void> { protected async doUpdate(ctx: TraceContext, updatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise<void> {
const span = TraceContext.startSpan("doUpdate", ctx); const span = TraceContext.startSpan("doUpdate", ctx);
try { try {
const githubApi = await this.getGitHubApi(Number.parseInt(updatatable.installationId)); const githubApi = await this.getGitHubApi(Number.parseInt(updatable.installationId));
if (!githubApi) { if (!githubApi) {
log.error("unable to authenticate GitHub app - this leaves user-facing checks dangling."); log.error("unable to authenticate GitHub app - this leaves user-facing checks dangling.");
return; return;
} }
const workspace = await this.workspaceDB.trace({span}).findById(pws.buildWorkspaceId); const workspace = await this.workspaceDB.trace({span}).findById(pws.buildWorkspaceId);
if (!!updatatable.contextUrl && !!workspace) { if (!!updatable.contextUrl && !!workspace) {
const conclusion = this.getConclusionFromPrebuildState(pws); const conclusion = this.getConclusionFromPrebuildState(pws);
if (conclusion === 'pending') { if (conclusion === 'pending') {
log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatatable.prebuiltWorkspaceId }); log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatable.prebuiltWorkspaceId });
return; return;
} }
let found = true; let found = true;
try { try {
await githubApi.repos.createCommitStatus({ await githubApi.repos.createCommitStatus({
owner: updatatable.owner, owner: updatable.owner,
repo: updatatable.repo, repo: updatable.repo,
context: "Gitpod", context: "Gitpod",
sha: pws.commit, sha: updatable.commitSHA || pws.commit,
target_url: updatatable.contextUrl, target_url: updatable.contextUrl,
description: conclusion === 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION, description: conclusion == 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION,
state: (workspace?.config?.github?.prebuilds?.addCheck === 'prevent-merge-on-error' ? conclusion : 'success') state: (workspace?.config?.github?.prebuilds?.addCheck === 'prevent-merge-on-error' ? conclusion : 'success')
}); });
} catch (err) { } catch (err) {
if (err.message == "Not Found") { if (err.message == "Not Found") {
log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatatable.owner, repo: updatatable.repo}); log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatable.owner, repo: updatable.repo});
found = true; found = true;
} else { } else {
throw err; throw err;
@ -185,10 +186,10 @@ export class PrebuildStatusMaintainer implements Disposable {
}, },
}); });
await this.workspaceDB.trace({span}).markUpdatableResolved(updatatable.id); await this.workspaceDB.trace({span}).markUpdatableResolved(updatable.id);
log.info(`Resolved updatable. Marked check on ${updatatable.contextUrl} as ${conclusion}`); log.info(`Resolved updatable. Marked check on ${updatable.contextUrl} as ${conclusion}`);
} else if (!!updatatable.issue) { } else if (!!updatable.issue) {
// this updatatable updates a label // this updatable updates a label
log.debug("Update label on a PR - we're not using this yet"); log.debug("Update label on a PR - we're not using this yet");
} }
} catch (err) { } catch (err) {

42
components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts
View File

@ -1,42 +0,0 @@
/**
* Copyright (c) 2021 Gitpod GmbH. All rights reserved.
* Licensed under the Gitpod Enterprise Source Code License,
* See License.enterprise.txt in the project root folder.
*/
import { User, WorkspaceContext, StartPrebuildContext, CommitContext, ContextURL } from "@gitpod/gitpod-protocol";
import { inject, injectable } from "inversify";
import { URL } from "url";
import { Config } from '../../../src/config';
import { HostContextProvider } from "../../../src/auth/host-context-provider";
import { IPrefixContextParser } from "../../../src/workspace/context-parser";
@injectable()
export class StartIncrementalPrebuildContextParser implements IPrefixContextParser {
@inject(Config) protected readonly config: Config;
@inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
static PREFIX = ContextURL.INCREMENTAL_PREBUILD_PREFIX + '/';
findPrefix(user: User, context: string): string | undefined {
if (context.startsWith(StartIncrementalPrebuildContextParser.PREFIX)) {
return StartIncrementalPrebuildContextParser.PREFIX;
}
}
public async handle(user: User, prefix: string, context: WorkspaceContext): Promise<WorkspaceContext> {
if (!CommitContext.is(context)) {
throw new Error("can only start incremental prebuilds on a commit context")
}
const host = new URL(context.repository.cloneUrl).hostname;
const hostContext = this.hostContextProvider.get(host);
const maxDepth = this.config.incrementalPrebuilds.commitHistory;
const result: StartPrebuildContext = {
title: `Prebuild of "${context.title}"`,
actual: context,
commitHistory: await (hostContext?.contextParser?.fetchCommitHistory({}, user, context.repository.cloneUrl, context.revision, maxDepth) || [])
};
return result;
}
}

37
components/server/ee/src/workspace/gitpod-server-impl.ts
View File

@ -661,18 +661,19 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
return; return;
} }
const commitSHAs = CommitContext.computeHash(context);
const logCtx: LogContext = { userId: user.id }; const logCtx: LogContext = { userId: user.id };
const cloneUrl = context.repository.cloneUrl; const cloneUrl = context.repository.cloneUrl;
// Note: findPrebuiltWorkspaceByCommit always returns the last triggered prebuild (so, if you re-trigger a prebuild, the newer one will always be used here) const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs);
const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision); const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace };
const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace };
log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload);
if (!prebuiltWorkspace) { if (!prebuiltWorkspace) {
return; return;
} }
if (prebuiltWorkspace.state === 'available') { if (prebuiltWorkspace.state === 'available') {
log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${context.revision}`, logPayload); log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${commitSHAs}`, logPayload);
const result: PrebuiltWorkspaceContext = { const result: PrebuiltWorkspaceContext = {
title: context.title, title: context.title,
originalContext: context, originalContext: context,
@ -752,7 +753,7 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
} else { } else {
result.runningWorkspacePrebuild!.starting = 'running'; result.runningWorkspacePrebuild!.starting = 'running';
} }
log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${context.revision}`, logPayload); log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${commitSHAs}`, logPayload);
return result; return result;
} }
} }
@ -766,27 +767,6 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
await this.licenseEvaluator.reloadLicense(); await this.licenseEvaluator.reloadLicense();
} }
// TODO(gpl) This is not part of our API interface, nor can I find any clients. Remove or re-surrect?
// async getLicenseInfo(ctx: TraceContext): Promise<GetLicenseInfoResult> {
// const user = this.checkAndBlockUser("getLicenseInfo");
// const { key } = await this.licenseKeySource.getKey();
// const { validUntil, seats } = this.licenseEvaluator.inspect();
// const { valid } = this.licenseEvaluator.validate();
// const isAdmin = this.authorizationService.hasPermission(user, Permission.ADMIN_API);
// return {
// isAdmin,
// licenseInfo: {
// key: isAdmin ? key : "REDACTED",
// seats,
// valid,
// validUntil
// }
// };
// }
async licenseIncludesFeature(ctx: TraceContext, licenseFeature: LicenseFeature): Promise<boolean> { async licenseIncludesFeature(ctx: TraceContext, licenseFeature: LicenseFeature): Promise<boolean> {
traceAPIParams(ctx, { licenseFeature }); traceAPIParams(ctx, { licenseFeature });
@ -1541,11 +1521,8 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
const context = await this.contextParser.handle(ctx, user, contextURL) as CommitContext; const context = await this.contextParser.handle(ctx, user, contextURL) as CommitContext;
const prebuild = await this.prebuildManager.startPrebuild(ctx, { const prebuild = await this.prebuildManager.startPrebuild(ctx, {
contextURL, context,
cloneURL: project.cloneUrl,
commit: context.revision,
user, user,
branch: branchDetails[0].name,
project project
}); });

179
components/server/ee/src/workspace/workspace-factory.ts
View File

@ -8,14 +8,13 @@ import { v4 as uuidv4 } from 'uuid';
import { WorkspaceFactory } from "../../../src/workspace/workspace-factory"; import { WorkspaceFactory } from "../../../src/workspace/workspace-factory";
import { injectable, inject } from "inversify"; import { injectable, inject } from "inversify";
import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, Project, PrebuiltWorkspace } from "@gitpod/gitpod-protocol"; import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, PrebuiltWorkspace, WorkspaceConfig, WorkspaceImageSource } from "@gitpod/gitpod-protocol";
import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { LicenseEvaluator } from '@gitpod/licensor/lib'; import { LicenseEvaluator } from '@gitpod/licensor/lib';
import { Feature } from '@gitpod/licensor/lib/api'; import { Feature } from '@gitpod/licensor/lib/api';
import { ResponseError } from 'vscode-jsonrpc'; import { ResponseError } from 'vscode-jsonrpc';
import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error'; import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error';
import { HostContextProvider } from '../../../src/auth/host-context-provider'; import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { RepoURL } from '../../../src/repohost';
@injectable() @injectable()
export class WorkspaceFactoryEE extends WorkspaceFactory { export class WorkspaceFactoryEE extends WorkspaceFactory {
@ -51,7 +50,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
const { project, branch } = context; const { project, branch } = context;
const commitContext: CommitContext = context.actual; const commitContext: CommitContext = context.actual;
const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, commitContext.revision); const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, CommitContext.computeHash(commitContext));
if (existingPWS) { if (existingPWS) {
const wsInstance = await this.db.trace({span}).findRunningInstance(existingPWS.buildWorkspaceId); const wsInstance = await this.db.trace({span}).findRunningInstance(existingPWS.buildWorkspaceId);
if (wsInstance) { if (wsInstance) {
@ -62,62 +61,28 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
const { config } = await this.configProvider.fetchConfig({span}, user, context.actual); const { config } = await this.configProvider.fetchConfig({span}, user, context.actual);
const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config); const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config);
// Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on. // Walk back the last prebuilds and check if they are valid ancestor.
let ws; let ws;
for (const parent of (context.commitHistory || [])) { if (context.commitHistory && context.commitHistory.length > 0) {
const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent); const recentPrebuilds = await this.db.trace({span}).findPrebuildsWithWorkpace(commitContext.repository.cloneUrl);
if (!parentPrebuild) { const match = recentPrebuilds.find(pb => this.isGoodBaseforIncrementalPrebuild(context, config, imageSource, pb.prebuild, pb.workspace));
continue; if (match) {
const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
title: `Incremental prebuild of "${commitContext.title}"`,
originalContext: commitContext,
prebuiltWorkspace: match.prebuild,
}
ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
// Overwrite the config from the parent prebuild:
// `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild.
// Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now.
// However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...)
// To fix this, we overwrite the new prebuild's config with the most-recently fetched config.
// See also: https://github.com/gitpod-io/gitpod/issues/7475
//TODO(sven) doing side effects on objects back and forth is complicated and error-prone. We should rather make sure we pass in the config when creating the prebuiltWorkspace.
ws.config = config;
} }
if (parentPrebuild.state !== 'available') {
continue;
}
log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild);
const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId);
if (!buildWorkspace) {
continue;
}
if (!!buildWorkspace.basedOnPrebuildId) {
continue;
}
if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) {
log.debug(`Skipping parent prebuild: Outdated image`, {
imageSource,
parentImageSource: buildWorkspace.imageSource,
});
continue;
}
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key))
// @ts-ignore
.reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
.filter(task => Object.keys(task).length > 0));
const prebuildTasks = filterPrebuildTasks(config.tasks);
const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks);
if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
prebuildTasks,
parentPrebuildTasks,
});
continue;
}
const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
title: `Incremental prebuild of "${commitContext.title}"`,
originalContext: commitContext,
prebuiltWorkspace: parentPrebuild,
}
ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
// Overwrite the config from the parent prebuild:
// `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild.
// Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now.
// However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...)
// To fix this, we overwrite the new prebuild's config with the most-recently fetched config.
// See also: https://github.com/gitpod-io/gitpod/issues/7475
ws.config = config;
break;
} }
if (!ws) { if (!ws) {
// No suitable parent prebuild was found -- create a (fresh) full prebuild. // No suitable parent prebuild was found -- create a (fresh) full prebuild.
ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL); ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
@ -130,21 +95,13 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
id: uuidv4(), id: uuidv4(),
buildWorkspaceId: ws.id, buildWorkspaceId: ws.id,
cloneURL: commitContext.repository.cloneUrl, cloneURL: commitContext.repository.cloneUrl,
commit: commitContext.revision, commit: CommitContext.computeHash(commitContext),
state: "queued", state: "queued",
creationTime: new Date().toISOString(), creationTime: new Date().toISOString(),
projectId: ws.projectId, projectId: ws.projectId,
branch branch
}); });
if (project) {
// do not await
this.storePrebuildInfo(ctx, project, pws, ws, user).catch(err => {
log.error(`failed to store prebuild info`, err);
TraceContext.setError({span}, err);
});
}
log.debug({ userId: user.id, workspaceId: ws.id }, `Registered workspace prebuild: ${pws.id} for ${commitContext.repository.cloneUrl}:${commitContext.revision}`); log.debug({ userId: user.id, workspaceId: ws.id }, `Registered workspace prebuild: ${pws.id} for ${commitContext.repository.cloneUrl}:${commitContext.revision}`);
return ws; return ws;
@ -156,43 +113,67 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
} }
} }
protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User) { private async isGoodBaseforIncrementalPrebuild(context: StartPrebuildContext, config: WorkspaceConfig, imageSource: WorkspaceImageSource, candidatePrebuild: PrebuiltWorkspace, candidate: Workspace) {
const span = TraceContext.startSpan("storePrebuildInfo", ctx); if (!context.commitHistory || context.commitHistory.length === 0) {
const { userId, teamId, name: projectName, id: projectId } = project; return false;
const parsedUrl = RepoURL.parseRepoUrl(project.cloneUrl);
if (!parsedUrl) {
return;
} }
const { owner, repo, host } = parsedUrl; if (!CommitContext.is(candidate.context)) {
const repositoryProvider = this.hostContextProvider.get(host)?.services?.repositoryProvider; return false;
if (!repositoryProvider) {
return;
} }
const commit = await repositoryProvider.getCommitInfo(user, owner, repo, pws.commit);
if (!commit) { // we are only considering available prebuilds
return; if (candidatePrebuild.state !== 'available') {
return false;
}
// we are only considering full prebuilds
if (!!candidate.basedOnPrebuildId) {
return false;
}
const candidateCtx = candidate.context;
if (candidateCtx.additionalRepositoryCheckoutInfo?.length !== context.additionalRepositoryCommitHistories?.length) {
// different number of repos
return false;
}
if (!context.commitHistory.some(sha => sha === candidateCtx.revision)) {
return false;
}
// check the commits are included in the commit history
for (const subRepo of candidateCtx.additionalRepositoryCheckoutInfo || []) {
const matchIngRepo = context.additionalRepositoryCommitHistories?.find(repo => repo.cloneUrl === subRepo.repository.cloneUrl);
if (!matchIngRepo || !matchIngRepo.commitHistory.some(sha => sha === subRepo.revision)) {
return false;
}
}
// ensure the image source hasn't changed
if (JSON.stringify(imageSource) !== JSON.stringify(candidate.imageSource)) {
log.debug(`Skipping parent prebuild: Outdated image`, {
imageSource,
parentImageSource: candidate.imageSource,
});
return false;
}
// ensure the tasks haven't changed
const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
.map(task => Object.keys(task)
.filter(key => ['before', 'init', 'prebuild'].includes(key))
// @ts-ignore
.reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
.filter(task => Object.keys(task).length > 0));
const prebuildTasks = filterPrebuildTasks(config.tasks);
const parentPrebuildTasks = filterPrebuildTasks(candidate.config.tasks);
if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
prebuildTasks,
parentPrebuildTasks,
});
return false;
} }
await this.db.trace({span}).storePrebuildInfo({
id: pws.id,
buildWorkspaceId: pws.buildWorkspaceId,
basedOnPrebuildId: ws.basedOnPrebuildId,
teamId,
userId,
projectName,
projectId,
startedAt: pws.creationTime,
startedBy: "", // TODO
startedByAvatar: "", // TODO
cloneUrl: pws.cloneURL,
branch: pws.branch || "unknown",
changeAuthor: commit.author,
changeAuthorAvatar: commit.authorAvatarUrl,
changeDate: commit.authorDate || "",
changeHash: commit.sha,
changeTitle: commit.commitMessage,
// changePR
changeUrl: ws.contextURL,
});
} }
protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise<Workspace> { protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise<Workspace> {

2
components/server/install-gh-app.sh
View File

@ -3,7 +3,6 @@
# This script will patch the servers config map, install the app cert and restart the server components # This script will patch the servers config map, install the app cert and restart the server components
# It is best to add the envs to your environment variables using `gp env GH_APP_ID=....` and `gp env GH_APP_KEY="..."`. # It is best to add the envs to your environment variables using `gp env GH_APP_ID=....` and `gp env GH_APP_KEY="..."`.
# See https://www.notion.so/gitpod/How-to-deploy-a-PR-with-a-working-GitHub-App-integration-d297a1ef2f7b4b3aa8483b2ae9b47da2 (internal) for more details. # See https://www.notion.so/gitpod/How-to-deploy-a-PR-with-a-working-GitHub-App-integration-d297a1ef2f7b4b3aa8483b2ae9b47da2 (internal) for more details.
# GH_APP_ID=<app-id> # GH_APP_ID=<app-id>
# GH_APP_KEY="-----BEGIN RSA PRIVATE KEY----- # GH_APP_KEY="-----BEGIN RSA PRIVATE KEY-----
# ... # ...
@ -32,6 +31,7 @@ kubectl get cm server-config -o yaml > server-config.yml
perl -0777 -i.original -pe "s/\"githubApp\":.+?\}/$LINE/igs" server-config.yml perl -0777 -i.original -pe "s/\"githubApp\":.+?\}/$LINE/igs" server-config.yml
kubectl apply -f server-config.yml kubectl apply -f server-config.yml
rm server-config.yml rm server-config.yml
rm server-config.yml.original
echo 'updating the secret' echo 'updating the secret'
kubectl delete secret server-github-app-cert kubectl delete secret server-github-app-cert

11
components/server/src/auth/host-context-provider.ts
View File

@ -6,6 +6,8 @@
import { HostContext } from "./host-context"; import { HostContext } from "./host-context";
import { AuthProviderParams } from "./auth-provider"; import { AuthProviderParams } from "./auth-provider";
import { CommitInfo, User } from "@gitpod/gitpod-protocol";
import { RepoURL } from "../repohost";
export const HostContextProvider = Symbol("HostContextProvider"); export const HostContextProvider = Symbol("HostContextProvider");
@ -16,6 +18,15 @@ export interface HostContextProvider {
findByAuthProviderId(authProviderId: string): HostContext | undefined; findByAuthProviderId(authProviderId: string): HostContext | undefined;
} }
export async function getCommitInfo(hostContextProvider: HostContextProvider, user: User, repoURL: string, commitSHA: string) {
const parsedRepo = RepoURL.parseRepoUrl(repoURL)!;
const hostCtx = hostContextProvider.get(parsedRepo.host);
let commitInfo: CommitInfo | undefined;
if (hostCtx?.services?.repositoryProvider) {
commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA);
}
return commitInfo;
}
export const HostContextProviderFactory = Symbol("HostContextProviderFactory"); export const HostContextProviderFactory = Symbol("HostContextProviderFactory");

59
components/server/src/auth/resource-access.ts
View File

@ -4,8 +4,10 @@
* See License-AGPL.txt in the project root for license information. * See License-AGPL.txt in the project root for license information.
*/ */
import { CommitContext, ContextURL, GitpodToken, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol"; import { CommitContext, GitpodToken, Repository, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
import { UnauthorizedError } from "../errors"; import { UnauthorizedError } from "../errors";
import { RepoURL } from "../repohost";
import { HostContextProvider } from "./host-context-provider"; import { HostContextProvider } from "./host-context-provider";
declare var resourceInstance: GuardedResource; declare var resourceInstance: GuardedResource;
@ -477,27 +479,40 @@ export class RepositoryResourceGuard implements ResourceAccessGuard {
// Check if user has at least read access to the repository // Check if user has at least read access to the repository
const workspace = resource.kind === 'snapshot' ? resource.workspace : resource.subject; const workspace = resource.kind === 'snapshot' ? resource.workspace : resource.subject;
const contextURL = ContextURL.getNormalizedURL(workspace); const repos: Repository[] = [];
if (!contextURL) { if (CommitContext.is(workspace.context)) {
throw new Error(`unable to parse ContextURL: ${contextURL}`); repos.push(workspace.context.repository);
for (const additionalRepo of workspace.context.additionalRepositoryCheckoutInfo || []) {
repos.push(additionalRepo.repository);
}
} }
const hostContext = this.hostContextProvider.get(contextURL.hostname); const result = await Promise.all(
if (!hostContext) { repos.map(
throw new Error(`no HostContext found for hostname: ${contextURL.hostname}`); async repo => {
} const repoUrl = RepoURL.parseRepoUrl(repo.cloneUrl);
const { authProvider } = hostContext; if (!repoUrl) {
const identity = User.getIdentity(this.user, authProvider.authProviderId); log.error("Cannot parse repoURL", {repo})
if (!identity) { return false;
throw UnauthorizedError.create(contextURL.hostname, authProvider.info.requirements?.default || [], "missing-identity"); }
} const hostContext = this.hostContextProvider.get(repoUrl.host)
const { services } = hostContext; if (!hostContext) {
if (!services) { throw new Error(`no HostContext found for hostname: ${repoUrl.host}`);
throw new Error(`no services found in HostContext for hostname: ${contextURL.hostname}`); }
} const { authProvider } = hostContext;
if (!CommitContext.is(workspace.context)) { const identity = User.getIdentity(this.user, authProvider.authProviderId);
return false; if (!identity) {
} throw UnauthorizedError.create(repoUrl!.host, authProvider.info.requirements?.default || [], "missing-identity");
const { owner, name: repo } = workspace.context.repository; }
return services.repositoryProvider.hasReadAccess(this.user, owner, repo); const { services } = hostContext;
if (!services) {
throw new Error(`no services found in HostContext for hostname: ${repoUrl.host}`);
}
if (!CommitContext.is(workspace.context)) {
return false;
}
return services.repositoryProvider.hasReadAccess(this.user, repo.owner, repo.name);
}
));
return result.every(b => b);
} }
} }

3
components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts
View File

@ -107,4 +107,7 @@ export class BitbucketServerRepositoryProvider implements RepositoryProvider {
return false; return false;
} }
async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise<string[]> {
return [];
}
} }

7
components/server/src/bitbucket/bitbucket-context-parser.spec.ts
View File

@ -471,13 +471,6 @@ class TestBitbucketContextParser {
"title": "gitpod/integration-tests-forked-repository - master" "title": "gitpod/integration-tests-forked-repository - master"
}) })
} }
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
expect(result).to.deep.equal([
'da2119f51b0e744cb6b36399f8433b477a4174ef',
])
}
} }
module.exports = new TestBitbucketContextParser(); module.exports = new TestBitbucketContextParser();

27
components/server/src/bitbucket/bitbucket-context-parser.ts
View File

@ -267,31 +267,4 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo
return result; return result;
} }
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[] | undefined> {
const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx);
try {
// TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
// The additional property 'page' may be helfpul.
const api = await this.api(user);
const { owner, repoName } = await this.parseURL(user, contextUrl);
const result = await api.repositories.listCommitsAt({
workspace: owner,
repo_slug: repoName,
revision: sha,
pagelen: maxDepth,
});
const commits = result.data.values?.slice(1);
if (!commits) {
return undefined;
}
return commits.map((v: Schema.Commit) => v.hash!);
} catch (e) {
span.log({ error: e });
log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e);
throw e;
} finally {
span.finish();
}
}
} }

6
components/server/src/bitbucket/bitbucket-repository-provider.spec.ts
View File

@ -72,6 +72,12 @@ class TestBitbucketRepositoryProvider {
}); });
} }
@test public async testFetchCommitHistory() {
const result = await this.repoProvider.getCommitHistory(this.user, 'gitpod', 'integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
expect(result).to.deep.equal([
'da2119f51b0e744cb6b36399f8433b477a4174ef',
])
}
} }
module.exports = new TestBitbucketRepositoryProvider(); module.exports = new TestBitbucketRepositoryProvider();

19
components/server/src/bitbucket/bitbucket-repository-provider.ts
View File

@ -5,6 +5,7 @@
*/ */
import { Branch, CommitInfo, Repository, User } from "@gitpod/gitpod-protocol"; import { Branch, CommitInfo, Repository, User } from "@gitpod/gitpod-protocol";
import { Schema } from "bitbucket";
import { inject, injectable } from 'inversify'; import { inject, injectable } from 'inversify';
import { URL } from "url"; import { URL } from "url";
import { RepoURL } from '../repohost/repo-url'; import { RepoURL } from '../repohost/repo-url';
@ -108,4 +109,22 @@ export class BitbucketRepositoryProvider implements RepositoryProvider {
// FIXME(janx): Not implemented yet // FIXME(janx): Not implemented yet
return false; return false;
} }
public async getCommitHistory(user: User, owner: string, repo: string, revision: string, maxDepth: number = 100): Promise<string[]> {
const api = await this.apiFactory.create(user);
// TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
// The additional property 'page' may be helfpul.
const result = await api.repositories.listCommitsAt({
workspace: owner,
repo_slug: repo,
revision: revision,
pagelen: maxDepth,
});
const commits = result.data.values?.slice(1);
if (!commits) {
return [];
}
return commits.map((v: Schema.Commit) => v.hash!);
}
} }

2
components/server/src/github/github-auth-provider.ts
View File

@ -69,7 +69,7 @@ export class GitHubAuthProvider extends GenericAuthProvider {
timeout: 5000, timeout: 5000,
}, },
userAgent: this.USER_AGENT, userAgent: this.USER_AGENT,
baseUrl: this.baseURL baseUrl: this.baseURL,
}); });
const fetchCurrentUser = async () => { const fetchCurrentUser = async () => {
const response = await api.users.getAuthenticated(); const response = await api.users.getAuthenticated();

9
components/server/src/github/github-context-parser.spec.ts
View File

@ -577,14 +577,5 @@ class TestGithubContextParser {
} }
) )
} }
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://github.com/gitpod-io/gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100);
expect(result).to.deep.equal([
'506e5aed317f28023994ecf8ca6ed91430e9c1a4',
'f5b041513bfab914b5fbf7ae55788d9835004d76',
])
}
} }
module.exports = new TestGithubContextParser() // Only to circumvent no usage warning :-/ module.exports = new TestGithubContextParser() // Only to circumvent no usage warning :-/

51
components/server/src/github/github-context-parser.ts
View File

@ -426,55 +426,4 @@ export class GithubContextParser extends AbstractContextParser implements IConte
`; `;
} }
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
const span = TraceContext.startSpan("GithubContextParser.fetchCommitHistory", ctx);
try {
if (sha.length != 40) {
throw new Error(`Invalid commit ID ${sha}.`);
}
// TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
// These additional history properties may be helfpul:
// totalCount,
// pageInfo {
// haxNextPage,
// },
const { owner, repoName } = await this.parseURL(user, contextUrl);
const result: any = await this.githubQueryApi.runQuery(user, `
query {
repository(name: "${repoName}", owner: "${owner}") {
object(oid: "${sha}") {
... on Commit {
history(first: ${maxDepth}) {
edges {
node {
oid
}
}
}
}
}
}
}
`);
span.log({"request.finished": ""});
if (result.data.repository === null) {
throw await NotFoundError.create(await this.tokenHelper.getCurrentToken(user), user, this.config.host, owner, repoName);
}
const commit = result.data.repository.object;
if (commit === null) {
throw new Error(`Couldn't find commit ${sha} in repository ${owner}/${repoName}.`);
}
return commit.history.edges.slice(1).map((e: any) => e.node.oid) || [];
} catch (e) {
span.log({"error": e});
throw e;
} finally {
span.finish();
}
}
} }

76
components/server/src/github/github-repository-provider.spec.ts Normal file
View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2020 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
// Use asyncIterators with es2015
if (typeof (Symbol as any).asyncIterator === 'undefined') {
(Symbol as any).asyncIterator = Symbol.asyncIterator || Symbol('asyncIterator');
}
import "reflect-metadata";
import { suite, test, timeout, retries } from "mocha-typescript";
import * as chai from 'chai';
const expect = chai.expect;
import { GitHubGraphQlEndpoint, GitHubRestApi } from './api';
import { User } from "@gitpod/gitpod-protocol";
import { ContainerModule, Container } from "inversify";
import { Config } from "../config";
import { DevData } from "../dev/dev-data";
import { AuthProviderParams } from "../auth/auth-provider";
import { TokenProvider } from "../user/token-provider";
import { GitHubTokenHelper } from "./github-token-helper";
import { HostContextProvider } from "../auth/host-context-provider";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { GithubRepositoryProvider } from "./github-repository-provider";
@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITHUB"))
class TestGithubContextRepositoryProvider {
protected provider: GithubRepositoryProvider;
protected user: User;
public before() {
const container = new Container();
container.load(new ContainerModule((bind, unbind, isBound, rebind) => {
bind(Config).toConstantValue({
// meant to appease DI, but Config is never actually used here
});
bind(GithubRepositoryProvider).toSelf().inSingletonScope();
bind(GitHubRestApi).toSelf().inSingletonScope();
bind(GitHubGraphQlEndpoint).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestGithubContextRepositoryProvider.AUTH_HOST_CONFIG);
bind(GitHubTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async (user: User, host: string) => {
return DevData.createGitHubTestToken();
}
});
bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider());
}));
this.provider = container.get(GithubRepositoryProvider);
this.user = DevData.createTestUser();
}
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitHub",
type: "GitHub",
verified: true,
description: "",
icon: "",
host: "github.com",
oauth: "not-used" as any
}
@test public async testFetchCommitHistory() {
const result = await this.provider.getCommitHistory(this.user, 'gitpod-io', 'gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100);
expect(result).to.deep.equal([
'506e5aed317f28023994ecf8ca6ed91430e9c1a4',
'f5b041513bfab914b5fbf7ae55788d9835004d76',
])
}
}
module.exports = new TestGithubContextRepositoryProvider() // Only to circumvent no usage warning :-/

54
components/server/src/github/github-repository-provider.ts
View File

@ -103,8 +103,58 @@ export class GithubRepositoryProvider implements RepositoryProvider {
} }
async getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined> { async getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined> {
const commit = await this.github.getCommit(user, { repo, owner, ref }); try {
return commit; return await this.github.getCommit(user, { repo, owner, ref });
} catch (error) {
console.error(error);
return undefined;
}
}
public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise<string[]> {
try {
if (ref.length != 40) {
throw new Error(`Invalid commit ID ${ref}.`);
}
// TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
// These additional history properties may be helfpul:
// totalCount,
// pageInfo {
// haxNextPage,
// },
const result: any = await this.githubQueryApi.runQuery(user, `
query {
repository(name: "${repo}", owner: "${owner}") {
object(oid: "${ref}") {
... on Commit {
history(first: ${maxDepth}) {
edges {
node {
oid
}
}
}
}
}
}
}
`);
if (result.data.repository === null) {
throw new Error(`couldn't find repository ${owner}/${repo} on ${this.github.baseURL}`);
}
const commit = result.data.repository.object;
if (commit === null) {
throw new Error(`Couldn't find commit ${ref} in repository ${owner}/${repo}.`);
}
return commit.history.edges.slice(1).map((e: any) => e.node.oid) || [];
} catch (e) {
console.error(e);
return [];
}
} }
async getUserRepos(user: User): Promise<string[]> { async getUserRepos(user: User): Promise<string[]> {

8
components/server/src/gitlab/gitlab-context-parser.spec.ts
View File

@ -604,14 +604,6 @@ class TestGitlabContextParser {
}) })
} }
@test public async testFetchCommitHistory() {
const result = await this.parser.fetchCommitHistory({}, this.user, 'https://gitlab.com/AlexTugarev/gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100);
expect(result).to.deep.equal([
'4447fbc4d46e6fd1ee41fb1b992702521ae078eb',
'f2d9790f2752a794517b949c65a773eb864844cd',
])
}
} }
module.exports = new TestGitlabContextParser(); module.exports = new TestGitlabContextParser();

19
components/server/src/gitlab/gitlab-context-parser.ts
View File

@ -394,23 +394,4 @@ export class GitlabContextParser extends AbstractContextParser implements IConte
}; };
} }
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
// TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled.
const { owner, repoName } = await this.parseURL(user, contextUrl);
const projectId = `${owner}/${repoName}`;
const result = await this.gitlabApi.run<GitLab.Commit[]>(user, async g => {
return g.Commits.all(projectId, {
ref_name: sha,
per_page: maxDepth,
page: 1,
});
});
if (GitLab.ApiError.is(result)) {
if (result.message === 'GitLab responded with code 404') {
throw new Error(`Couldn't find commit #${sha} in repository ${projectId}.`);
}
throw result;
}
return result.slice(1).map((c: GitLab.Commit) => c.id);
}
} }

64
components/server/src/gitlab/gitlab-repository-provider.spec.ts Normal file
View File

@ -0,0 +1,64 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/
import { User } from "@gitpod/gitpod-protocol";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { expect } from "chai";
import { Container, ContainerModule } from "inversify";
import { suite, retries, test, timeout } from "mocha-typescript";
import { AuthProviderParams } from "../auth/auth-provider";
import { HostContextProvider } from "../auth/host-context-provider";
import { DevData } from "../dev/dev-data";
import { TokenProvider } from "../user/token-provider";
import { GitLabApi } from "./api";
import { GitlabContextParser } from "./gitlab-context-parser";
import { GitlabRepositoryProvider } from "./gitlab-repository-provider";
import { GitLabTokenHelper } from "./gitlab-token-helper";
@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITLAB"))
class TestGitlabRepositoryProvider {
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitLab",
type: "GitLab",
verified: true,
description: "",
icon: "",
host: "gitlab.com",
}
protected repositoryProvider: GitlabRepositoryProvider;
protected user: User;
public before() {
const container = new Container();
container.load(new ContainerModule((bind, unbind, isBound, rebind) => {
bind(GitlabContextParser).toSelf().inSingletonScope();
bind(GitLabApi).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestGitlabRepositoryProvider.AUTH_HOST_CONFIG);
bind(GitLabTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async () => DevData.createGitlabTestToken(),
getFreshPortAuthenticationToken: async (user: User, workspaceId: string) => DevData.createPortAuthTestToken(workspaceId),
});
bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider());
bind(GitlabRepositoryProvider).toSelf().inSingletonScope();
}));
this.repositoryProvider = container.get(GitlabRepositoryProvider);
this.user = DevData.createTestUser();
}
@test public async testFetchCommitHistory() {
const result = await this.repositoryProvider.getCommitHistory(this.user, 'AlexTugarev', 'gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100);
expect(result).to.deep.equal([
'4447fbc4d46e6fd1ee41fb1b992702521ae078eb',
'f2d9790f2752a794517b949c65a773eb864844cd',
])
}
}
module.exports = new TestGitlabRepositoryProvider();

19
components/server/src/gitlab/gitlab-repository-provider.ts
View File

@ -113,4 +113,23 @@ export class GitlabRepositoryProvider implements RepositoryProvider {
return false; return false;
} }
} }
public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise<string[]> {
// TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled.
const projectId = `${owner}/${repo}`;
const result = await this.gitlab.run<GitLab.Commit[]>(user, async g => {
return g.Commits.all(projectId, {
ref_name: ref,
per_page: maxDepth,
page: 1,
});
});
if (GitLab.ApiError.is(result)) {
if (result.message === 'GitLab responded with code 404') {
throw new Error(`Couldn't find commit #${ref} in repository ${projectId}.`);
}
throw result;
}
return result.slice(1).map((c: GitLab.Commit) => c.id);
}
} }

4
components/server/src/gitlab/gitlab-token-helper.ts
View File

@ -31,8 +31,8 @@ export class GitLabTokenHelper {
if (this.containsScopes(token, requiredScopes)) { if (this.containsScopes(token, requiredScopes)) {
return token; return token;
} }
} catch { } catch (e) {
// no token console.error(e);
} }
if (requiredScopes.length === 0) { if (requiredScopes.length === 0) {
requiredScopes = GitLabScope.Requirements.DEFAULT requiredScopes = GitLabScope.Requirements.DEFAULT

4
components/server/src/projects/projects-service.ts
View File

@ -94,9 +94,7 @@ export class ProjectsService {
changeHash: commit.sha, changeHash: commit.sha,
changeTitle: commit.commitMessage, changeTitle: commit.commitMessage,
changeAuthorAvatar: commit.authorAvatarUrl, changeAuthorAvatar: commit.authorAvatarUrl,
isDefault: repository.defaultBranch === branch.name, isDefault: repository.defaultBranch === branch.name
changePR: "changePR", // todo: compute in repositoryProvider
changeUrl: "changeUrl", // todo: compute in repositoryProvider
}); });
} }
result.sort((a, b) => (b.changeDate || "").localeCompare(a.changeDate || "")); result.sort((a, b) => (b.changeDate || "").localeCompare(a.changeDate || ""));

1
components/server/src/repohost/repository-provider.ts
View File

@ -15,4 +15,5 @@ export interface RepositoryProvider {
getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined>; getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise<CommitInfo | undefined>;
getUserRepos(user: User): Promise<string[]>; getUserRepos(user: User): Promise<string[]>;
hasReadAccess(user: User, owner: string, repo: string): Promise<boolean>; hasReadAccess(user: User, owner: string, repo: string): Promise<boolean>;
getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise<string[]>;
} }

109
components/server/src/workspace/context-parser-service.ts
View File

@ -65,21 +65,25 @@ export class ContextParser {
protected async internalHandleWithoutPrefix(ctx: TraceContext, user: User, nonPrefixedContextURL: string): Promise<WorkspaceContext> { protected async internalHandleWithoutPrefix(ctx: TraceContext, user: User, nonPrefixedContextURL: string): Promise<WorkspaceContext> {
const span = TraceContext.startSpan("ContextParser.internalHandle", ctx); const span = TraceContext.startSpan("ContextParser.internalHandle", ctx);
let result: WorkspaceContext | undefined; try {
let result: WorkspaceContext | undefined;
for (const parser of this.allContextParsers) { for (const parser of this.allContextParsers) {
if (parser.canHandle(user, nonPrefixedContextURL)) { if (parser.canHandle(user, nonPrefixedContextURL)) {
result = await parser.handle({ span }, user, nonPrefixedContextURL); result = await parser.handle({ span }, user, nonPrefixedContextURL);
break; break;
}
}
if (!result) {
throw new Error(`Couldn't parse context '${nonPrefixedContextURL}'.`);
} }
}
if (!result) {
throw new Error(`Couldn't parse context '${nonPrefixedContextURL}'.`);
}
// TODO: Make the parsers return the context with normalizedContextURL set // TODO: Make the parsers return the context with normalizedContextURL set
result.normalizedContextURL = nonPrefixedContextURL; result.normalizedContextURL = nonPrefixedContextURL;
return result; return result;
} finally {
span.finish();
}
} }
protected buildUpstreamCloneUrl(context: CommitContext): string | undefined { protected buildUpstreamCloneUrl(context: CommitContext): string | undefined {
@ -101,47 +105,52 @@ export class ContextParser {
return context; return context;
} }
const span = TraceContext.startSpan("ContextParser.handleMultiRepositoryContext", ctx); const span = TraceContext.startSpan("ContextParser.handleMultiRepositoryContext", ctx);
let config = await this.configProvider.fetchConfig({ span }, user, context); try {
let mainRepoContext: WorkspaceContext | undefined; let config = await this.configProvider.fetchConfig({ span }, user, context);
if (config.config.mainRepository) { let mainRepoContext: WorkspaceContext | undefined;
mainRepoContext = await this.internalHandleWithoutPrefix({ span }, user, config.config.mainRepository); if (config.config.mainConfiguration) {
if (!CommitContext.is(mainRepoContext)) { mainRepoContext = await this.internalHandleWithoutPrefix({ span }, user, config.config.mainConfiguration);
throw new InvalidGitpodYMLError([`Cannot find main repository '${config.config.mainRepository}'.`]); if (!CommitContext.is(mainRepoContext)) {
} throw new InvalidGitpodYMLError([`Cannot find main repository '${config.config.mainConfiguration}'.`]);
config = await this.configProvider.fetchConfig({ span }, user, mainRepoContext);
}
if (config.config.subRepositories && config.config.subRepositories.length > 0) {
const subRepoCommits: GitCheckoutInfo[] = [];
for (const subRepo of config.config.subRepositories) {
let subContext = await this.internalHandleWithoutPrefix({ span }, user, subRepo.url) as CommitContext;
if (!CommitContext.is(subContext)) {
throw new InvalidGitpodYMLError([`Cannot find sub-repository '${subRepo.url}'.`]);
} }
if (context.repository.cloneUrl === subContext.repository.cloneUrl) { config = await this.configProvider.fetchConfig({ span }, user, mainRepoContext);
// if it's the repo from the original context we want to use that commit.
subContext = JSON.parse(JSON.stringify(context));
}
subRepoCommits.push({
... subContext,
checkoutLocation: (subRepo.checkoutLocation || subContext.repository.name),
upstreamRemoteURI: this.buildUpstreamCloneUrl(subContext),
localBranch: context.localBranch // we want to create a local branch on all repos, in case it's a multi-repo change. If it's not there are no drawbacks anyway.
});
} }
context.subRepositoryCheckoutInfo = subRepoCommits;
if (config.config.additionalRepositories && config.config.additionalRepositories.length > 0) {
const subRepoCommits: GitCheckoutInfo[] = [];
for (const subRepo of config.config.additionalRepositories) {
let subContext = await this.internalHandleWithoutPrefix({ span }, user, subRepo.url) as CommitContext;
if (!CommitContext.is(subContext)) {
throw new InvalidGitpodYMLError([`Cannot find sub-repository '${subRepo.url}'.`]);
}
if (context.repository.cloneUrl === subContext.repository.cloneUrl) {
// if it's the repo from the original context we want to use that commit.
subContext = JSON.parse(JSON.stringify(context));
}
subRepoCommits.push({
... subContext,
checkoutLocation: (subRepo.checkoutLocation || subContext.repository.name),
upstreamRemoteURI: this.buildUpstreamCloneUrl(subContext),
// we want to create a local branch on all repos, in case it's a multi-repo change. If it's not there are no drawbacks anyway.
ref: context.ref,
refType: context.refType,
localBranch: context.localBranch
});
}
context.additionalRepositoryCheckoutInfo = subRepoCommits;
}
// if the original contexturl was pointing to a subrepo we update the commit information with the mainContext.
if (mainRepoContext && CommitContext.is(mainRepoContext)) {
context.repository = mainRepoContext.repository;
context.revision = mainRepoContext.revision;
}
context.checkoutLocation = (config.config.checkoutLocation || context.repository.name);
context.upstreamRemoteURI = this.buildUpstreamCloneUrl(context);
return context;
} finally {
span.finish();
} }
// if the original contexturl was pointing to a subrepo we update the commit information with the mainContext.
if (mainRepoContext && CommitContext.is(mainRepoContext)) {
context.repository = mainRepoContext.repository;
context.revision = mainRepoContext.revision;
context.ref = mainRepoContext.revision;
context.refType = mainRepoContext.refType;
}
context.checkoutLocation = (config.config.checkoutLocation || context.repository.name);
context.upstreamRemoteURI = this.buildUpstreamCloneUrl(context);
return context;
} }
protected findPrefix(user: User, context: string): { prefix: string, parser: IPrefixContextParser } | undefined { protected findPrefix(user: User, context: string): { prefix: string, parser: IPrefixContextParser } | undefined {

8
components/server/src/workspace/context-parser.ts
View File

@ -14,7 +14,6 @@ export interface IContextParser {
normalize?(contextUrl: string): string | undefined normalize?(contextUrl: string): string | undefined
canHandle(user: User, contextUrl: string): boolean canHandle(user: User, contextUrl: string): boolean
handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext> handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>
fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise<string[] | undefined>
} }
export const IContextParser = Symbol("IContextParser") export const IContextParser = Symbol("IContextParser")
@ -78,13 +77,6 @@ export abstract class AbstractContextParser implements IContextParser {
} }
public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>; public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>;
/**
* Fetches the commit history of a commit (used to find a relevant parent prebuild for incremental prebuilds).
*
* @returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log`
*/
public abstract fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise<string[] | undefined>;
} }
export interface URLParts { export interface URLParts {

18
components/server/src/workspace/workspace-starter.ts
View File

@ -7,7 +7,7 @@
import { CloneTargetMode, FileDownloadInitializer, GitAuthMethod, GitConfig, GitInitializer, PrebuildInitializer, SnapshotInitializer, WorkspaceInitializer } from "@gitpod/content-service/lib"; import { CloneTargetMode, FileDownloadInitializer, GitAuthMethod, GitConfig, GitInitializer, PrebuildInitializer, SnapshotInitializer, WorkspaceInitializer } from "@gitpod/content-service/lib";
import { CompositeInitializer, FromBackupInitializer } from "@gitpod/content-service/lib/initializer_pb"; import { CompositeInitializer, FromBackupInitializer } from "@gitpod/content-service/lib/initializer_pb";
import { DBUser, DBWithTracing, ProjectDB, TracedUserDB, TracedWorkspaceDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib'; import { DBUser, DBWithTracing, ProjectDB, TracedUserDB, TracedWorkspaceDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib';
import { CommitContext, Disposable, GitpodToken, GitpodTokenType, IssueContext, NamedWorkspaceFeatureFlag, PullRequestContext, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ImageBuildLogInfo, ProjectEnvVar } from "@gitpod/gitpod-protocol"; import { CommitContext, Disposable, GitpodToken, GitpodTokenType, GitCheckoutInfo, NamedWorkspaceFeatureFlag, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ProjectEnvVar, ImageBuildLogInfo } from "@gitpod/gitpod-protocol";
import { IAnalyticsWriter } from '@gitpod/gitpod-protocol/lib/analytics'; import { IAnalyticsWriter } from '@gitpod/gitpod-protocol/lib/analytics';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
@ -1016,9 +1016,13 @@ export class WorkspaceStarter {
const init = new PrebuildInitializer(); const init = new PrebuildInitializer();
init.setPrebuild(snapshot); init.setPrebuild(snapshot);
if (initializer instanceof CompositeInitializer) { if (initializer instanceof CompositeInitializer) {
init.setComposite(initializer); for (const myInit of initializer.getInitializerList()) {
if (myInit instanceof WorkspaceInitializer && myInit.hasGit()) {
init.addGit(myInit.getGit());
}
}
} else { } else {
init.setGit(initializer); init.addGit(initializer);
} }
result.setPrebuild(init); result.setPrebuild(init);
} else if (WorkspaceProbeContext.is(context)) { } else if (WorkspaceProbeContext.is(context)) {
@ -1073,13 +1077,13 @@ export class WorkspaceStarter {
protected async createCommitInitializer(ctx: TraceContext, workspace: Workspace, context: CommitContext, user: User): Promise<{initializer: GitInitializer | CompositeInitializer, disposable: Disposable}> { protected async createCommitInitializer(ctx: TraceContext, workspace: Workspace, context: CommitContext, user: User): Promise<{initializer: GitInitializer | CompositeInitializer, disposable: Disposable}> {
const span = TraceContext.startSpan("createInitializerForCommit", ctx); const span = TraceContext.startSpan("createInitializerForCommit", ctx);
const mainGit = this.createGitInitializer({ span }, workspace, context, user); const mainGit = this.createGitInitializer({ span }, workspace, context, user);
if (!context.subRepositoryCheckoutInfo || context.subRepositoryCheckoutInfo.length === 0) { if (!context.additionalRepositoryCheckoutInfo || context.additionalRepositoryCheckoutInfo.length === 0) {
return mainGit; return mainGit;
} }
const subRepoInitializers = [mainGit]; const subRepoInitializers = [mainGit];
await Promise.all(context.subRepositoryCheckoutInfo.map(async subRepo => { for (const subRepo of context.additionalRepositoryCheckoutInfo) {
subRepoInitializers.push(this.createGitInitializer({ span }, workspace, subRepo , user)); subRepoInitializers.push(this.createGitInitializer({ span }, workspace, subRepo , user));
})); }
const inits = await Promise.all(subRepoInitializers); const inits = await Promise.all(subRepoInitializers);
const compositeInit = new CompositeInitializer(); const compositeInit = new CompositeInitializer();
const compositeDisposable = new DisposableCollection(); const compositeDisposable = new DisposableCollection();

6
components/ws-daemon/pkg/content/service.go
View File

@ -282,8 +282,10 @@ func getCheckoutLocation(req *api.InitWorkspaceRequest) string {
} }
} }
if ir, ok := spec.(*csapi.WorkspaceInitializer_Prebuild); ok { if ir, ok := spec.(*csapi.WorkspaceInitializer_Prebuild); ok {
if ir.Prebuild != nil && ir.Prebuild.Git != nil { if ir.Prebuild != nil {
return ir.Prebuild.Git.CheckoutLocation if len(ir.Prebuild.Git) > 0 {
return ir.Prebuild.Git[0].CheckoutLocation
}
} }
} }
return "" return ""

3
scripts/mysql.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
kubectl port-forward statefulset/mysql 3306:3306 &
mysql -h 127.0.0.1 -P 3306 -u gitpod -D gitpod --select-limit=200 --safe-updates --password="$(kubectl get secrets mysql -o jsonpath="{.data.password}" | base64 -d)"