package deployer import ( "context" "fmt" "log/slog" "maps" "strconv" "strings" "sync" "time" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" "tailscale.com/util/truncate" ) type DeployStatus struct { Status string Message string } const ( statusProgress = "progress" statusError = "error" statusSuccess = "success" ) func (d *DockerDeployer) StartDeploy(challenge, team string) string { deploymentKey, job := d.createDeployJob(challenge, team) go func() { err := d.doDeployment(deploymentKey, job, context.Background()) if err != nil { d.writeDeployChannel(deploymentKey, DeployStatus{Status: statusError, Message: ``}) } }() return deploymentKey } func (d *DockerDeployer) doDeployment(deployKey string, job *DeployJob, ctx context.Context) error { log := slog.With( "deployment", deployKey, "challenge", job.challenge, "team", job.team, ) var unlockOnce sync.Once // if !d.containerCreateMutex.TryLock() { // d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Challenge deployment queued"}) // d.containerCreateMutex.Lock() // } d.containerCreateMutex.Lock() defer unlockOnce.Do(d.containerCreateMutex.Unlock) log.Info("starting challenge deployment") imageName := fmt.Sprintf("%s%s:latest", d.imagePrefix, job.challenge) if err := d.containerExistsForTeam(ctx, log, job); err != nil { return err } d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Pulling image"}) if err := d.pullImage(ctx, log, imageName); err != nil { return err } d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Reading challenge configuration"}) imageCfg, err := d.readImageMetadata(ctx, log, imageName) if err != nil { return err } proxyRouteName := strings.ReplaceAll(deployKey, "-", "") url := truncate.String(job.challenge, 30) + "-" + proxyRouteName + "." + d.instancerDomain expiry := time.Now().Add(90 * time.Minute) d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Configuring network"}) networkID, err := d.setupNetwork(ctx, log, job, expiry, proxyRouteName, imageCfg.ProxyEnable) if err != nil { return err } log.Info("challenge network created", "networkID", networkID) d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Creating container"}) containerID, err := d.createContainer(ctx, log, job, expiry, imageName, proxyRouteName, networkID, url, imageCfg) if err != nil { return err } unlockOnce.Do(d.containerCreateMutex.Unlock) log.Info("challenge container created", "containerID", containerID) d.writeDeployChannel(deployKey, DeployStatus{Status: statusProgress, Message: "Starting container"}) if _, err := d.client.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); err != nil { log.Error("could not start challenge container", "cause", err) return fmt.Errorf("error starting container") } d.writeDeployChannel(deployKey, DeployStatus{Status: statusSuccess, Message: ``}) return nil } func (d *DockerDeployer) containerExistsForTeam(ctx context.Context, log *slog.Logger, job *DeployJob) error { filters := client.Filters{} filters.Add("label", ContainerLabelChallenge+"="+job.challenge) filters.Add("label", ContainerLabelForTeam+"="+job.team) containers, err := d.client.ContainerList(ctx, client.ContainerListOptions{ All: true, Filters: filters, }) if err != nil { log.Error("could not pull list challenges", "cause", err) return fmt.Errorf("error checking uniqueness") } if len(containers.Items) > 0 { return fmt.Errorf("instance of challenge already deployed for team") } return nil } func (d *DockerDeployer) pullImage(ctx context.Context, log *slog.Logger, imageName string) error { resp, err := d.client.ImagePull(ctx, imageName, client.ImagePullOptions{ RegistryAuth: d.registryLogin(), }) if err != nil { log.Error("could not pull challenge image", "cause", err) return fmt.Errorf("error pulling challenge image") } defer resp.Close() resp.Wait(ctx) return nil } func (d *DockerDeployer) readImageMetadata(ctx context.Context, log *slog.Logger, imageName string) (*ImageConfig, error) { inspect, err := d.client.ImageInspect(ctx, imageName) if err != nil { log.Error("could not inspect image", "cause", err) return nil, fmt.Errorf("error loading challenge configuration") } imageCfg, err := extractImageConfig(inspect.Config.Labels) if err != nil { log.Error("invalid challenge configuration", "cause", err) return nil, fmt.Errorf("error loading challenge configuration") } return &imageCfg, nil } func (d *DockerDeployer) setupNetwork(ctx context.Context, log *slog.Logger, job *DeployJob, expiry time.Time, routeName string, proxy bool) (string, error) { resp, err := d.client.NetworkCreate(ctx, "n-"+routeName, client.NetworkCreateOptions{ Driver: "bridge", Labels: map[string]string{ ContainerLabelManaged: "yes", ContainerLabelCreatedAt: strconv.FormatInt(time.Now().Unix(), 10), ContainerLabelExpiresAt: strconv.FormatInt(expiry.Unix(), 10), ContainerLabelDeployKey: job.deployKey, ContainerLabelChallenge: job.challenge, ContainerLabelForTeam: job.team, }, }) if err != nil { log.Error("could not create challenge network", "cause", err) return "", fmt.Errorf("could not create challenge network") } if proxy { if _, err := d.client.NetworkConnect(ctx, resp.ID, client.NetworkConnectOptions{ Container: d.proxyContainerName, }); err != nil { log.Error("could not connect proxy to challenge network", "cause", err) return "", fmt.Errorf("could not connect proxy to challenge network") } } return resp.ID, nil } func (d *DockerDeployer) createContainer(ctx context.Context, log *slog.Logger, job *DeployJob, expiry time.Time, imageName, routeName, networkID, url string, imageCfg *ImageConfig) (string, error) { now := time.Now() labels := map[string]string{ ContainerLabelManaged: "yes", ContainerLabelCreatedAt: strconv.FormatInt(now.Unix(), 10), ContainerLabelExpiresAt: strconv.FormatInt(expiry.Unix(), 10), ContainerLabelDeployKey: job.deployKey, ContainerLabelChallenge: job.challenge, ContainerLabelForTeam: job.team, ContainerLabelAddress: url, } if imageCfg.ProxyEnable { //TODO do tcp maps.Copy(labels, map[string]string{ ContainerLabelAddressFormat: "https", "traefik.enable": "true", "traefik.docker.network": "n-" + routeName, "traefik.http.routers." + routeName + ".entrypoints": "websecure", "traefik.http.routers." + routeName + ".rule": "Host(`" + url + "`)", "traefik.http.routers." + routeName + ".tls": "true", "traefik.http.routers." + routeName + ".service": routeName, "traefik.http.services." + routeName + ".loadbalancer.server.port": strconv.Itoa(imageCfg.Port), }) } resp, err := d.client.ContainerCreate(ctx, client.ContainerCreateOptions{ Image: imageName, Config: &container.Config{ Labels: labels, }, Name: "c-" + routeName, HostConfig: &container.HostConfig{ Resources: container.Resources{ Memory: imageCfg.Limits.Memory, NanoCPUs: imageCfg.Limits.CPU, }, ReadonlyRootfs: imageCfg.Security.ReadOnlyFS, SecurityOpt: imageCfg.Security.SecurityOpt, CapAdd: imageCfg.Security.CapAdd, CapDrop: imageCfg.Security.CapDrop, }, NetworkingConfig: &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ networkID: {}, }, }, }) if err != nil { log.Error("could not create challenge container", "cause", err) return "", fmt.Errorf("error creating container") } return resp.ID, nil }