gitpod/components/ws-manager/pkg/manager/monitor_integration_test.go

// Copyright (c) 2020 TypeFox GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

package manager

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/gitpod-io/gitpod/common-go/log"
	"github.com/gitpod-io/gitpod/common-go/tracing"
	wsdaemon "github.com/gitpod-io/gitpod/ws-daemon/api"
	wsdaemon_mock "github.com/gitpod-io/gitpod/ws-daemon/api/mock"
	"github.com/gitpod-io/gitpod/ws-manager/api"
	"github.com/golang/mock/gomock"
	"github.com/sirupsen/logrus"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	corev1 "k8s.io/api/core/v1"
)

func TestIntegrationWorkspaceDisposal(t *testing.T) {
	log.Log.Logger.SetLevel(logrus.PanicLevel)
	tracing.Init("TestWorkspaceDisposal")

	regularWorkspaceTemplates := IntegrationTestPodTemplates{
		Default: &corev1.Pod{
			Spec: corev1.PodSpec{
				Containers: []corev1.Container{
					{
						Name: "workspace",
						ReadinessProbe: &corev1.Probe{
							Handler: corev1.Handler{
								Exec: &corev1.ExecAction{Command: []string{"echo"}},
							},
						},
					},
				},
			},
		},
	}

	tests := []struct {
		Desc string
		Skip string
		T    *SingleWorkspaceIntegrationTest
	}{
		{
			Desc: "failed image pull",
			T: &SingleWorkspaceIntegrationTest{
				StartRequestModifier: func(t *testing.T, r *api.StartWorkspaceRequest) {
					r.Spec.WorkspaceImage = "does-not-exist"
				},
				MockWsdaemon: func(t *testing.T, s *wsdaemon_mock.MockWorkspaceContentServiceServer) {
					s.EXPECT().InitWorkspace(gomock.Any(), gomock.Any()).DoAndReturn(func(a, b interface{}) { time.Sleep(1 * time.Second) }).Return(&wsdaemon.InitWorkspaceResponse{}, nil)
					s.EXPECT().WaitForInit(gomock.Any(), gomock.Any()).Return(&wsdaemon.WaitForInitResponse{}, nil).AnyTimes()
					s.EXPECT().DisposeWorkspace(gomock.Any(), matches(func(a interface{}) bool {
						req, ok := a.(*wsdaemon.DisposeWorkspaceRequest)
						if !ok {
							return false
						}
						if req.Backup {
							t.Logf("req.Backup == true but should have been false")
						}
						// currently this test is just way too flakey. We should fix this!
						// return req.Backup == false
						return true
					})).Return(&wsdaemon.DisposeWorkspaceResponse{}, nil).MinTimes(1)
				},
				PostStart: func(t *testing.T, monitor *Monitor, id string, updates *StatusRecoder) {
					ok := updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						return s.Phase == api.WorkspacePhase_STOPPED
					}, 30*time.Second)
					if !ok {
						t.Log(updates.String())
						t.Fatalf("workspace did not stop in time")
					}
				},
			},
		},
		{
			Desc: "failed init",
			T: &SingleWorkspaceIntegrationTest{
				StartRequestModifier: func(t *testing.T, r *api.StartWorkspaceRequest) {
					r.Spec.WorkspaceImage = "gitpod/workspace-full"
				},
				MockWsdaemon: func(t *testing.T, s *wsdaemon_mock.MockWorkspaceContentServiceServer) {
					s.EXPECT().InitWorkspace(gomock.Any(), gomock.Any()).DoAndReturn(func(a, b interface{}) { time.Sleep(1 * time.Second) }).Return(nil, status.Error(codes.Internal, "fail intentionally"))
					s.EXPECT().WaitForInit(gomock.Any(), gomock.Any()).Return(&wsdaemon.WaitForInitResponse{}, nil).AnyTimes()
					s.EXPECT().DisposeWorkspace(gomock.Any(), matches(func(a interface{}) bool {
						_, ok := a.(*wsdaemon.DisposeWorkspaceRequest)
						if !ok {
							return false
						}
						return true //req.Backup == false
					})).Return(&wsdaemon.DisposeWorkspaceResponse{}, nil).MinTimes(1)
				},
				PostStart: func(t *testing.T, monitor *Monitor, id string, updates *StatusRecoder) {
					ok := updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						return s.Phase == api.WorkspacePhase_STOPPED
					}, 30*time.Second)
					if !ok {
						t.Log(updates.String())
						t.Fatalf("workspace did not stop in time")
					}
				},
			},
		},
		{
			Desc: "regular workspace - all fine",
			T: &SingleWorkspaceIntegrationTest{
				PodTemplates: regularWorkspaceTemplates,
				StartRequestModifier: func(t *testing.T, s *api.StartWorkspaceRequest) {
					s.Spec.WorkspaceImage = "csweichel/noop:latest"
				},
				MockWsdaemon: func(t *testing.T, s *wsdaemon_mock.MockWorkspaceContentServiceServer) {
					initCall := s.EXPECT().InitWorkspace(gomock.Any(), gomock.Any()).Return(&wsdaemon.InitWorkspaceResponse{}, nil)
					s.EXPECT().WaitForInit(gomock.Any(), gomock.Any()).Return(&wsdaemon.WaitForInitResponse{}, nil).MinTimes(1).After(initCall)
					s.EXPECT().DisposeWorkspace(gomock.Any(), matches(func(a interface{}) bool {
						req, ok := a.(*wsdaemon.DisposeWorkspaceRequest)
						if !ok {
							return false
						}
						return req.Backup == true
					})).Return(&wsdaemon.DisposeWorkspaceResponse{}, nil).MinTimes(1)
				},
				PostStart: func(t *testing.T, monitor *Monitor, id string, updates *StatusRecoder) {
					ok := updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						if s.Conditions.Failed != "" {
							t.Fatalf("workspace failed: %s", s.Conditions.Failed)
						}

						return s.Phase == api.WorkspacePhase_RUNNING
					}, 20*time.Second)
					if !ok {
						t.Errorf("workspace did not start in time")
						t.Log(updates.String())
					}

					err := monitor.manager.stopWorkspace(context.Background(), id, 10*time.Second)
					if err != nil {
						t.Fatalf("cannot stop workspace: %q", err)
					}

					ok = updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						return s.Phase == api.WorkspacePhase_STOPPED
					}, 60*time.Second)
					if !ok {
						t.Log(updates.String())
						t.Fatalf("workspace did not stop in time")
					}
				},
			},
		},
		{
			Desc: "regular workspace - dispose timeout",
			T: &SingleWorkspaceIntegrationTest{
				PodTemplates: regularWorkspaceTemplates,
				StartRequestModifier: func(t *testing.T, s *api.StartWorkspaceRequest) {
					s.Spec.WorkspaceImage = "csweichel/noop:latest"
				},
				MockWsdaemon: func(t *testing.T, s *wsdaemon_mock.MockWorkspaceContentServiceServer) {
					initCall := s.EXPECT().InitWorkspace(gomock.Any(), gomock.Any()).Return(&wsdaemon.InitWorkspaceResponse{}, nil)
					s.EXPECT().WaitForInit(gomock.Any(), gomock.Any()).Return(&wsdaemon.WaitForInitResponse{}, nil).MinTimes(1).After(initCall)
					s.EXPECT().DisposeWorkspace(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, req interface{}) (resp *wsdaemon.DisposeWorkspaceRequest, err error) {
						return nil, context.DeadlineExceeded
					}).MinTimes(wsdaemonMaxAttempts)
				},
				PostStart: func(t *testing.T, monitor *Monitor, id string, updates *StatusRecoder) {
					ok := updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						if s.Conditions.Failed != "" {
							t.Fatalf("workspace failed: %s", s.Conditions.Failed)
						}

						return s.Phase == api.WorkspacePhase_RUNNING
					}, 20*time.Second)
					if !ok {
						t.Errorf("workspace did not start in time")
						t.Log(updates.String())
					}

					err := monitor.manager.stopWorkspace(context.Background(), id, 10*time.Second)
					if err != nil {
						t.Fatalf("cannot stop workspace: %q", err)
					}

					ok = updates.WaitFor(func(s *api.WorkspaceStatus) bool {
						return s.Phase == api.WorkspacePhase_STOPPED
					}, 60*time.Second)
					if !ok {
						t.Log(updates.String())
						t.Fatalf("workspace did not stop in time")
					}

					var stoppedStatus *api.WorkspaceStatus
					for _, status := range updates.Log() {
						if status.Phase != api.WorkspacePhase_STOPPED {
							continue
						}
						stoppedStatus = &status
						break
					}
					if stoppedStatus == nil {
						t.Fatalf("did not record stopped status although the workspace was stopped. StatusRecoder bug?")
					}
					if !strings.Contains(stoppedStatus.Conditions.Failed, "last backup failed") {
						t.Errorf("unexpected workspace failure mode: %+q", stoppedStatus)
					}
				},
			},
		},
	}

	for _, test := range tests {
		t.Run(test.Desc, func(t *testing.T) {
			if test.Skip != "" {
				t.Skip(test.Skip)
				return
			}

			test.T.FillDefaults().Run(t)
		})
	}
}

func matches(p func(interface{}) bool) gomock.Matcher {
	return &funcMatcher{p}
}

type funcMatcher struct {
	P func(a interface{}) bool
}

func (f *funcMatcher) Matches(a interface{}) bool {
	return f.P(a)
}

func (f *funcMatcher) String() string {
	return "matches predicate"
}