gitpod/components/image-builder-mk3/pkg/orchestrator/orchestrator.go

// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

package orchestrator

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/docker/distribution/reference"
	"github.com/google/uuid"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
	"golang.org/x/xerrors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/encoding/protojson"

	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
	"github.com/gitpod-io/gitpod/common-go/log"
	"github.com/gitpod-io/gitpod/common-go/tracing"
	csapi "github.com/gitpod-io/gitpod/content-service/api"
	"github.com/gitpod-io/gitpod/image-builder/api"
	protocol "github.com/gitpod-io/gitpod/image-builder/api"
	"github.com/gitpod-io/gitpod/image-builder/api/config"
	"github.com/gitpod-io/gitpod/image-builder/pkg/auth"
	"github.com/gitpod-io/gitpod/image-builder/pkg/resolve"
	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
)

const (
	// buildWorkspaceManagerID identifies the manager for the workspace
	buildWorkspaceManagerID = "image-builder"

	// maxBuildRuntime is the maximum time a build is allowed to take
	maxBuildRuntime = 60 * time.Minute

	// workspaceBuildProcessVersion controls how we build workspace images.
	// Incrementing this value will trigger a rebuild of all workspace images.
	workspaceBuildProcessVersion = 2
)

// NewOrchestratingBuilder creates a new orchestrating image builder
func NewOrchestratingBuilder(cfg config.Configuration) (res *Orchestrator, err error) {
	var authentication auth.RegistryAuthenticator
	if cfg.PullSecretFile != "" {
		fn := cfg.PullSecretFile
		if tproot := os.Getenv("TELEPRESENCE_ROOT"); tproot != "" {
			fn = filepath.Join(tproot, fn)
		}

		authentication, err = auth.NewDockerConfigFileAuth(fn)
		if err != nil {
			return
		}
	}

	var wsman wsmanapi.WorkspaceManagerClient
	if c, ok := cfg.WorkspaceManager.Client.(wsmanapi.WorkspaceManagerClient); ok {
		wsman = c
	} else {
		grpcOpts := common_grpc.DefaultClientOptions()
		if cfg.WorkspaceManager.TLS.Authority != "" || cfg.WorkspaceManager.TLS.Certificate != "" && cfg.WorkspaceManager.TLS.PrivateKey != "" {
			tlsConfig, err := common_grpc.ClientAuthTLSConfig(
				cfg.WorkspaceManager.TLS.Authority, cfg.WorkspaceManager.TLS.Certificate, cfg.WorkspaceManager.TLS.PrivateKey,
				common_grpc.WithSetRootCAs(true),
				common_grpc.WithServerName("ws-manager"),
			)
			if err != nil {
				log.WithField("config", cfg.WorkspaceManager.TLS).Error("Cannot load ws-manager certs - this is a configuration issue.")
				return nil, xerrors.Errorf("cannot load ws-manager certs: %w", err)
			}

			grpcOpts = append(grpcOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
		} else {
			grpcOpts = append(grpcOpts, grpc.WithTransportCredentials(insecure.NewCredentials()))
		}
		conn, err := grpc.Dial(cfg.WorkspaceManager.Address, grpcOpts...)
		if err != nil {
			return nil, err
		}
		wsman = wsmanapi.NewWorkspaceManagerClient(conn)
	}

	o := &Orchestrator{
		Config: cfg,
		Auth:   authentication,
		AuthResolver: auth.Resolver{
			BaseImageRepository:      cfg.BaseImageRepository,
			WorkspaceImageRepository: cfg.WorkspaceImageRepository,
		},
		RefResolver: &resolve.StandaloneRefResolver{},

		wsman:         wsman,
		buildListener: make(map[string]map[buildListener]struct{}),
		logListener:   make(map[string]map[logListener]struct{}),
		censorship:    make(map[string][]string),
		metrics:       newMetrics(),
	}
	o.monitor = newBuildMonitor(o, o.wsman)

	return o, nil
}

// Orchestrator runs image builds by orchestrating headless build workspaces
type Orchestrator struct {
	Config       config.Configuration
	Auth         auth.RegistryAuthenticator
	AuthResolver auth.Resolver
	RefResolver  resolve.DockerRefResolver

	wsman wsmanapi.WorkspaceManagerClient

	buildListener map[string]map[buildListener]struct{}
	logListener   map[string]map[logListener]struct{}
	censorship    map[string][]string
	mu            sync.RWMutex

	monitor *buildMonitor

	metrics *metrics

	protocol.UnimplementedImageBuilderServer
}

// Start fires up the internals of this image builder
func (o *Orchestrator) Start(ctx context.Context) error {
	go o.monitor.Run()
	return nil
}

// ResolveBaseImage returns the "digest" form of a Docker image tag thereby making it absolute.
func (o *Orchestrator) ResolveBaseImage(ctx context.Context, req *protocol.ResolveBaseImageRequest) (resp *protocol.ResolveBaseImageResponse, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "ResolveBaseImage")
	defer tracing.FinishSpan(span, &err)
	tracing.LogRequestSafe(span, req)

	reqs, _ := protojson.Marshal(req)
	safeReqs, _ := log.RedactJSON(reqs)
	log.WithField("req", string(safeReqs)).Debug("ResolveBaseImage")

	reqauth := o.AuthResolver.ResolveRequestAuth(req.Auth)

	refstr, err := o.getAbsoluteImageRef(ctx, req.Ref, reqauth)
	if err != nil {
		return nil, err
	}

	return &protocol.ResolveBaseImageResponse{
		Ref: refstr,
	}, nil
}

// ResolveWorkspaceImage returns information about a build configuration without actually attempting to build anything.
func (o *Orchestrator) ResolveWorkspaceImage(ctx context.Context, req *protocol.ResolveWorkspaceImageRequest) (resp *protocol.ResolveWorkspaceImageResponse, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "ResolveWorkspaceImage")
	defer tracing.FinishSpan(span, &err)
	tracing.LogRequestSafe(span, req)

	reqs, _ := protojson.Marshal(req)
	safeReqs, _ := log.RedactJSON(reqs)
	log.WithField("req", string(safeReqs)).Debug("ResolveWorkspaceImage")

	reqauth := o.AuthResolver.ResolveRequestAuth(req.Auth)
	baseref, err := o.getBaseImageRef(ctx, req.Source, reqauth)
	if _, ok := status.FromError(err); err != nil && ok {
		return nil, err
	}
	if err != nil {
		return nil, status.Errorf(codes.Internal, "cannot resolve base image: %s", err.Error())
	}
	refstr, err := o.getWorkspaceImageRef(ctx, baseref)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "cannot produce image ref: %v", err)
	}
	span.LogKV("refstr", refstr, "baseref", baseref)

	// to check if the image exists we must have access to the image caching registry and the refstr we check here does not come
	// from the user. Thus we can safely use auth.AllowedAuthForAll here.
	auth, err := auth.AllowedAuthForAll().GetAuthFor(o.Auth, refstr)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "cannot get workspace image authentication: %v", err)
	}
	exists, err := o.checkImageExists(ctx, refstr, auth)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "cannot resolve workspace image: %s", err.Error())
	}

	var status protocol.BuildStatus
	if exists {
		status = protocol.BuildStatus_done_success
	} else {
		status = protocol.BuildStatus_unknown
	}

	return &protocol.ResolveWorkspaceImageResponse{
		Status: status,
		Ref:    refstr,
	}, nil
}

// Build initiates the build of a Docker image using a build configuration. If a build of this
// configuration is already ongoing no new build will be started.
func (o *Orchestrator) Build(req *protocol.BuildRequest, resp protocol.ImageBuilder_BuildServer) (err error) {
	span, ctx := opentracing.StartSpanFromContext(resp.Context(), "Build")
	defer tracing.FinishSpan(span, &err)
	tracing.LogRequestSafe(span, req)

	if req.Source == nil {
		return status.Errorf(codes.InvalidArgument, "build source is missing")
	}

	// resolve build request authentication
	reqauth := o.AuthResolver.ResolveRequestAuth(req.Auth)

	baseref, err := o.getBaseImageRef(ctx, req.Source, reqauth)
	if _, ok := status.FromError(err); err != nil && ok {
		return err
	}
	if err != nil {
		return status.Errorf(codes.Internal, "cannot resolve base image: %s", err.Error())
	}
	wsrefstr, err := o.getWorkspaceImageRef(ctx, baseref)
	if err != nil {
		return status.Errorf(codes.Internal, "cannot produce workspace image ref: %q", err)
	}
	wsrefAuth, err := auth.AllowedAuthForAll().GetAuthFor(o.Auth, wsrefstr)
	if err != nil {
		return status.Errorf(codes.Internal, "cannot get workspace image authentication: %q", err)
	}

	// check if needs build -> early return
	exists, err := o.checkImageExists(ctx, wsrefstr, wsrefAuth)
	if err != nil {
		return status.Errorf(codes.Internal, "cannot check if image is already built: %q", err)
	}
	if exists && !req.GetForceRebuild() {
		// If the workspace image exists, so should the baseimage if we've built it.
		// If we didn't build it and the base image doesn't exist anymore, getWorkspaceImageRef will have failed to resolve the baseref.
		baserefAbsolute, err := o.getAbsoluteImageRef(ctx, baseref, auth.AllowedAuthForAll())
		if err != nil {
			return err
		}

		// image has already been built - no need for us to start building
		err = resp.Send(&protocol.BuildResponse{
			Status:  protocol.BuildStatus_done_success,
			Ref:     wsrefstr,
			BaseRef: baserefAbsolute,
		})
		if err != nil {
			return err
		}
		return nil
	}

	o.metrics.BuildStarted()

	// Once a build is running we don't want it cancelled becuase the server disconnected i.e. during deployment.
	// Instead we want to impose our own timeout/lifecycle on the build. Using context.WithTimeout does not shadow its parent's
	// cancelation (see https://play.golang.org/p/N3QBIGlp8Iw for an example/experiment).
	ctx, cancel := context.WithTimeout(&parentCantCancelContext{Delegate: ctx}, maxBuildRuntime)
	defer cancel()

	randomUUID, err := uuid.NewRandom()
	if err != nil {
		return
	}

	var (
		buildID        = randomUUID.String()
		buildBase      = "false"
		contextPath    = "."
		dockerfilePath = "Dockerfile"
	)
	var initializer *csapi.WorkspaceInitializer = &csapi.WorkspaceInitializer{
		Spec: &csapi.WorkspaceInitializer_Empty{
			Empty: &csapi.EmptyInitializer{},
		},
	}
	if fsrc := req.Source.GetFile(); fsrc != nil {
		buildBase = "true"
		initializer = fsrc.Source
		contextPath = fsrc.ContextPath
		dockerfilePath = fsrc.DockerfilePath
	}
	dockerfilePath = filepath.Join("/workspace", dockerfilePath)

	if contextPath == "" {
		contextPath = filepath.Dir(dockerfilePath)
	}
	contextPath = filepath.Join("/workspace", strings.TrimPrefix(contextPath, "/workspace"))

	o.censor(buildID, []string{
		wsrefstr,
		baseref,
		strings.Split(wsrefstr, ":")[0],
		strings.Split(baseref, ":")[0],
	})

	// push some log to the client before starting the job, just in case the build workspace takes a while to start up
	o.PublishLog(buildID, "starting image build")

	retryIfUnavailable1 := func(err error) bool {
		if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable {
			return true
		}
		return false
	}

	pbaseref, err := reference.Parse(baseref)
	if err != nil {
		return xerrors.Errorf("cannot parse baseref: %v", err)
	}
	bobBaseref := "localhost:8080/base"
	if r, ok := pbaseref.(reference.Digested); ok {
		bobBaseref += "@" + r.Digest().String()
	} else {
		bobBaseref += ":latest"
	}
	wsref, err := reference.ParseNamed(wsrefstr)
	var additionalAuth []byte
	if err == nil {
		additionalAuth, err = json.Marshal(reqauth.GetImageBuildAuthFor([]string{
			reference.Domain(wsref),
		}))
		if err != nil {
			return xerrors.Errorf("cannot marshal additional auth: %w", err)
		}
	}

	var swr *wsmanapi.StartWorkspaceResponse
	err = retry(ctx, func(ctx context.Context) (err error) {
		swr, err = o.wsman.StartWorkspace(ctx, &wsmanapi.StartWorkspaceRequest{
			Id:            buildID,
			ServicePrefix: buildID,
			Metadata: &wsmanapi.WorkspaceMetadata{
				MetaId: buildID,
				Annotations: map[string]string{
					annotationRef:       wsrefstr,
					annotationBaseRef:   baseref,
					annotationManagedBy: buildWorkspaceManagerID,
				},
				Owner: req.GetTriggeredBy(),
			},
			Spec: &wsmanapi.StartWorkspaceSpec{
				Initializer:        initializer,
				Timeout:            maxBuildRuntime.String(),
				WorkspaceImage:     o.Config.BuilderImage,
				DeprecatedIdeImage: o.Config.BuilderImage,
				IdeImage: &wsmanapi.IDEImage{
					WebRef: o.Config.BuilderImage,
				},
				WorkspaceLocation: contextPath,
				Envvars: []*wsmanapi.EnvironmentVariable{
					{Name: "BOB_TARGET_REF", Value: "localhost:8080/target:latest"},
					{Name: "BOB_BASE_REF", Value: bobBaseref},
					{Name: "BOB_BUILD_BASE", Value: buildBase},
					{Name: "BOB_DOCKERFILE_PATH", Value: dockerfilePath},
					{Name: "BOB_CONTEXT_DIR", Value: contextPath},
					{Name: "GITPOD_TASKS", Value: `[{"name": "build", "init": "sudo -E /app/bob build"}]`},
					{Name: "WORKSPACEKIT_RING2_ENCLAVE", Value: "/app/bob proxy"},
					{Name: "WORKSPACEKIT_BOBPROXY_BASEREF", Value: baseref},
					{Name: "WORKSPACEKIT_BOBPROXY_TARGETREF", Value: wsrefstr},
					{
						Name: "WORKSPACEKIT_BOBPROXY_AUTH",
						Secret: &wsmanapi.EnvironmentVariable_SecretKeyRef{
							SecretName: o.Config.PullSecret,
							Key:        ".dockerconfigjson",
						},
					},
					{
						Name:  "WORKSPACEKIT_BOBPROXY_ADDITIONALAUTH",
						Value: string(additionalAuth),
					},
					{Name: "SUPERVISOR_DEBUG_ENABLE", Value: fmt.Sprintf("%v", log.Log.Logger.IsLevelEnabled(logrus.DebugLevel))},
				},
			},
			Type: wsmanapi.WorkspaceType_IMAGEBUILD,
		})
		return
	}, retryIfUnavailable1, 1*time.Second, 10)
	if status.Code(err) == codes.AlreadyExists {
		// build is already running - do not add it to the list of builds
	} else if errors.Is(err, errOutOfRetries) {
		return status.Error(codes.Unavailable, "workspace services are currently unavailable")
	} else if err != nil {
		return status.Errorf(codes.Internal, "cannot start build: %q", err)
	} else {
		o.monitor.RegisterNewBuild(buildID, wsrefstr, baseref, swr.Url, swr.OwnerToken)
		o.PublishLog(buildID, "starting image build ...\n")
	}

	updates, cancel := o.registerBuildListener(buildID)
	defer cancel()
	for {
		update := <-updates
		if update == nil {
			// channel was closed unexpectatly
			return status.Error(codes.Aborted, "subscription canceled - please try again")
		}

		// The failed condition of ws-manager is not stable, hence we might wrongly report that the
		// build was successful when in fact it wasn't. This would break workspace startup with a strange
		// "cannot pull from reg.gitpod.io" error message. Instead the image-build should fail properly.
		// To do this, we resolve the built image afterwards to ensure it was actually built.
		if update.Status == protocol.BuildStatus_done_success {
			exists, err := o.checkImageExists(ctx, wsrefstr, wsrefAuth)
			if err != nil {
				update.Status = protocol.BuildStatus_done_failure
				update.Message = fmt.Sprintf("cannot check if workspace image exists after the build: %v", err)
			} else if !exists {
				update.Status = protocol.BuildStatus_done_failure
				update.Message = "image build did not produce a workspace image"
			}
		}

		err := resp.Send(update)
		if err != nil {
			log.WithError(err).Error("cannot forward build update - dropping listener")
			return status.Errorf(codes.Unknown, "cannot send update: %v", err)
		}

		if update.Status == protocol.BuildStatus_done_failure || update.Status == protocol.BuildStatus_done_success {
			// build is done
			o.clearListener(buildID)
			break
		}
	}

	return nil
}

// publishStatus broadcasts a build status update to all listeners
func (o *Orchestrator) PublishStatus(buildID string, resp *api.BuildResponse) {
	o.mu.RLock()
	listener, ok := o.buildListener[buildID]
	o.mu.RUnlock()

	// we don't have any log listener for this build
	if !ok {
		return
	}

	log.WithField("buildID", buildID).WithField("resp", resp).Debug("publishing status")

	for l := range listener {
		select {
		case l <- resp:
			continue

		case <-time.After(5 * time.Second):
			log.Warn("timeout while forwarding status to listener - dropping listener")
			o.mu.Lock()
			ll := o.buildListener[buildID]
			// In the meantime the listener list may have been removed/cleared by a call to clearListener.
			// We don't have to do any work in this case.
			if ll != nil {
				close(l)
				delete(ll, l)
			}
			o.mu.Unlock()
		}
	}
}

// Logs listens to the build output of an ongoing Docker build identified build the build ID
func (o *Orchestrator) Logs(req *protocol.LogsRequest, resp protocol.ImageBuilder_LogsServer) (err error) {
	span, ctx := opentracing.StartSpanFromContext(resp.Context(), "Logs")
	defer tracing.FinishSpan(span, &err)
	tracing.LogRequestSafe(span, req)

	rb, err := o.monitor.GetAllRunningBuilds(ctx)
	var buildID string
	for _, bld := range rb {
		if bld.Info.Ref == req.BuildRef {
			buildID = bld.Info.BuildId
			break
		}
	}
	if buildID == "" {
		return status.Error(codes.NotFound, "build not found")
	}

	logs, cancel := o.registerLogListener(buildID)
	defer cancel()
	for {
		update := <-logs
		if update == nil {
			break
		}

		err := resp.Send(update)
		if err != nil {
			log.WithError(err).Error("cannot forward log output - dropping listener")
			return status.Errorf(codes.Unknown, "cannot send log output: %v", err)
		}
	}

	return
}

// ListBuilds returns a list of currently running builds
func (o *Orchestrator) ListBuilds(ctx context.Context, req *protocol.ListBuildsRequest) (resp *protocol.ListBuildsResponse, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "ListBuilds")
	defer tracing.FinishSpan(span, &err)

	builds, err := o.monitor.GetAllRunningBuilds(ctx)
	if err != nil {
		return
	}

	res := make([]*protocol.BuildInfo, 0, len(builds))
	for _, ws := range builds {
		res = append(res, &ws.Info)
	}

	return &protocol.ListBuildsResponse{Builds: res}, nil
}

func (o *Orchestrator) checkImageExists(ctx context.Context, ref string, authentication *auth.Authentication) (exists bool, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "checkImageExists")
	defer tracing.FinishSpan(span, &err)
	span.SetTag("ref", ref)

	_, err = o.RefResolver.Resolve(ctx, ref, resolve.WithAuthentication(authentication))
	if errors.Is(err, resolve.ErrNotFound) {
		return false, nil
	}
	if errors.Is(err, resolve.ErrUnauthorized) {
		return false, status.Errorf(codes.Unauthenticated, "cannot check if image exists: %q", err)
	}
	if err != nil {
		return false, err
	}

	return true, nil
}

// getAbsoluteImageRef returns the "digest" form of an image, i.e. contains no mutable image tags
func (o *Orchestrator) getAbsoluteImageRef(ctx context.Context, ref string, allowedAuth auth.AllowedAuthFor) (res string, err error) {
	auth, err := allowedAuth.GetAuthFor(o.Auth, ref)
	if err != nil {
		return "", status.Errorf(codes.InvalidArgument, "cannt resolve base image ref: %v", err)
	}

	ref, err = o.RefResolver.Resolve(ctx, ref, resolve.WithAuthentication(auth))
	if xerrors.Is(err, resolve.ErrNotFound) {
		return "", status.Error(codes.NotFound, "cannot resolve image")
	}
	if xerrors.Is(err, resolve.ErrUnauthorized) {
		return "", status.Error(codes.Unauthenticated, "cannot resolve image")
	}
	if err != nil {
		return "", status.Errorf(codes.Internal, "cannot resolve image: %v", err)
	}
	return ref, nil
}

func (o *Orchestrator) getBaseImageRef(ctx context.Context, bs *protocol.BuildSource, allowedAuth auth.AllowedAuthFor) (res string, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "getBaseImageRef")
	defer tracing.FinishSpan(span, &err)

	switch src := bs.From.(type) {
	case *protocol.BuildSource_Ref:
		return o.getAbsoluteImageRef(ctx, src.Ref.Ref, allowedAuth)

	case *protocol.BuildSource_File:
		manifest := map[string]string{
			"DockerfilePath":    src.File.DockerfilePath,
			"DockerfileVersion": src.File.DockerfileVersion,
			"ContextPath":       src.File.ContextPath,
		}
		// workspace starter will only ever send us Git sources. Should that ever change, we'll need to add
		// manifest support for the other initializer types.
		if src.File.Source.GetGit() != nil {
			fsrc := src.File.Source.GetGit()
			manifest["Source"] = "git"
			manifest["CloneTarget"] = fsrc.CloneTaget
			manifest["RemoteURI"] = fsrc.RemoteUri
		} else {
			return "", xerrors.Errorf("unsupported context initializer")
		}
		// Go maps do NOT maintain their order - we must sort the keys to maintain a stable order
		var keys []string
		for k := range manifest {
			keys = append(keys, k)
		}
		sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
		var dfl string
		for _, k := range keys {
			dfl += fmt.Sprintf("%s: %s\n", k, manifest[k])
		}
		span.LogKV("manifest", dfl)

		hash := sha256.New()
		n, err := hash.Write([]byte(dfl))
		if err != nil {
			return "", xerrors.Errorf("cannot compute src image ref: %w", err)
		}
		if n < len(dfl) {
			return "", xerrors.Errorf("cannot compute src image ref: short write")
		}

		// the mkII image builder supported an image hash salt. That salt broke other assumptions,
		// which is why this mkIII implementation does not support it anymore. We need to stay compatible
		// with the previous means of computing the hash though. This is why we add an extra breakline here,
		// basically defaulting to an empty salt string.
		_, err = fmt.Fprintln(hash, "")
		if err != nil {
			return "", xerrors.Errorf("cannot compute src image ref: %w", err)
		}

		return fmt.Sprintf("%s:%x", o.Config.BaseImageRepository, hash.Sum([]byte{})), nil

	default:
		return "", xerrors.Errorf("invalid base image")
	}
}

func (o *Orchestrator) getWorkspaceImageRef(ctx context.Context, baseref string) (ref string, err error) {
	cnt := []byte(fmt.Sprintf("%s\n%d\n", baseref, workspaceBuildProcessVersion))
	hash := sha256.New()
	n, err := hash.Write(cnt)
	if err != nil {
		return "", xerrors.Errorf("cannot produce workspace image name: %w", err)
	}
	if n < len(cnt) {
		return "", xerrors.Errorf("cannot produce workspace image name: %w", io.ErrShortWrite)
	}

	dst := hash.Sum([]byte{})
	return fmt.Sprintf("%s:%x", o.Config.WorkspaceImageRepository, dst), nil
}

// parentCantCancelContext is a bit of a hack. We have some operations which we want to keep alive even after clients
// disconnect. gRPC cancels the context once a client disconnects, thus we intercept the cancelation and act as if
// nothing had happened.
//
// This cannot be the best way to do this. Ideally we'd like to intercept client disconnect, but maintain the usual
// cancelation mechanism such as deadlines, timeouts, explicit cancelation.
type parentCantCancelContext struct {
	Delegate context.Context
	done     chan struct{}
}

func (*parentCantCancelContext) Deadline() (deadline time.Time, ok bool) {
	// return ok==false which means there's no deadline set
	return time.Time{}, false
}

func (c *parentCantCancelContext) Done() <-chan struct{} {
	return c.done
}

func (c *parentCantCancelContext) Err() error {
	err := c.Delegate.Err()
	if err == context.Canceled {
		return nil
	}

	return err
}

func (c *parentCantCancelContext) Value(key interface{}) interface{} {
	return c.Delegate.Value(key)
}

type buildListener chan *api.BuildResponse

type logListener chan *api.LogsResponse

func (o *Orchestrator) registerBuildListener(buildID string) (c <-chan *api.BuildResponse, cancel func()) {
	o.mu.Lock()
	defer o.mu.Unlock()

	l := make(buildListener)
	ls := o.buildListener[buildID]
	if ls == nil {
		ls = make(map[buildListener]struct{})
	}
	ls[l] = struct{}{}
	o.buildListener[buildID] = ls

	cancel = func() {
		o.mu.Lock()
		defer o.mu.Unlock()
		ls := o.buildListener[buildID]
		if ls == nil {
			return
		}
		delete(ls, l)
		o.buildListener[buildID] = ls
	}
	return l, cancel
}

func (o *Orchestrator) registerLogListener(buildID string) (c <-chan *api.LogsResponse, cancel func()) {
	o.mu.Lock()
	defer o.mu.Unlock()

	l := make(logListener)
	ls := o.logListener[buildID]
	if ls == nil {
		ls = make(map[logListener]struct{})
	}
	ls[l] = struct{}{}
	o.logListener[buildID] = ls
	log.WithField("buildID", buildID).WithField("listener", len(ls)).Debug("registered log listener")

	cancel = func() {
		o.mu.Lock()
		defer o.mu.Unlock()
		ls := o.logListener[buildID]
		if ls == nil {
			return
		}
		delete(ls, l)
		o.logListener[buildID] = ls

		log.WithField("buildID", buildID).WithField("listener", len(ls)).Debug("deregistered log listener")
	}
	return l, cancel
}

// clearListener removes all listener for a particular build
func (o *Orchestrator) clearListener(buildID string) {
	o.mu.Lock()
	defer o.mu.Unlock()

	delete(o.buildListener, buildID)
	delete(o.logListener, buildID)
	delete(o.censorship, buildID)
}

// censor registers tokens that are censored in the log output
func (o *Orchestrator) censor(buildID string, words []string) {
	o.mu.Lock()
	defer o.mu.Unlock()

	o.censorship[buildID] = words
}

// PublishLog broadcasts log output to all registered listener
func (o *Orchestrator) PublishLog(buildID string, message string) {
	o.mu.RLock()
	listener, ok := o.logListener[buildID]
	o.mu.RUnlock()

	// we don't have any log listener for this build
	if !ok {
		return
	}

	o.mu.RLock()
	wds := o.censorship[buildID]
	o.mu.RUnlock()
	for _, w := range wds {
		message = strings.ReplaceAll(message, w, "")
	}

	for l := range listener {
		select {
		case l <- &api.LogsResponse{
			Content: []byte(message),
		}:
			continue

		case <-time.After(5 * time.Second):
			log.WithField("buildID", buildID).Warn("timeout while forwarding log to listener - dropping listener")
			o.mu.Lock()
			ll := o.logListener[buildID]
			// In the meantime the listener list may have been removed/cleared by a call to clearListener.
			// We don't have to do any work in this case.
			if ll != nil {
				close(l)
				delete(ll, l)
			}
			o.mu.Unlock()
		}
	}
}