package deployer
import (
"context"
"fmt"
"log/slog"
"maps"
"strconv"
"strings"
"sync"
"time"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
"tailscale.com/util/truncate"
)
type DeployStatus struct {
Status string
Message string
}
const (
statusProgress = "progress"
statusError = "error"
statusSuccess = "success"
)
func (d *DockerDeployer) StartDeploy(challenge, team string) string {
deploymentKey, job := d.createDeployJob(challenge, team)
go func() {
err := d.doDeployment(deploymentKey, job, context.Background())
if err != nil {
d.writeDeployChannel(deploymentKey, DeployStatus{Status: statusError, Message: `
Aborted: ` + err.Error() + `
`})
}
}()
return deploymentKey
}
func (d *DockerDeployer) doDeployment(deployKey string, job *DeployJob, ctx context.Context) error {
log := slog.With(
"deployment", deployKey,
"challenge", job.challenge,
"team", job.team,
)
var unlockOnce sync.Once
// if !d.containerCreateMutex.TryLock() {
// d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Challenge deployment queued"})
// d.containerCreateMutex.Lock()
// }
d.containerCreateMutex.Lock()
defer unlockOnce.Do(d.containerCreateMutex.Unlock)
log.Info("starting challenge deployment")
imageName := fmt.Sprintf("%s%s:latest", d.imagePrefix, job.challenge)
if err := d.containerExistsForTeam(ctx, log, job); err != nil {
return err
}
d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Pulling image"})
if err := d.pullImage(ctx, log, imageName); err != nil {
return err
}
d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Reading challenge configuration"})
imageCfg, err := d.readImageMetadata(ctx, log, imageName)
if err != nil {
return err
}
proxyRouteName := strings.ReplaceAll(deployKey, "-", "")
url := truncate.String(job.challenge, 30) + "-" + proxyRouteName + "." + d.instancerDomain
expiry := time.Now().Add(90 * time.Minute)
d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Configuring network"})
networkID, err := d.setupNetwork(ctx, log, job, expiry, proxyRouteName, imageCfg.ProxyEnable)
if err != nil {
return err
}
log.Info("challenge network created", "networkID", networkID)
d.writeDeployChannel(deployKey, DeployStatus{statusProgress, "Creating container"})
containerID, err := d.createContainer(ctx, log, job, expiry, imageName, proxyRouteName, networkID, url, imageCfg)
if err != nil {
return err
}
unlockOnce.Do(d.containerCreateMutex.Unlock)
log.Info("challenge container created", "containerID", containerID)
d.writeDeployChannel(deployKey, DeployStatus{Status: statusProgress, Message: "Starting container"})
if _, err := d.client.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); err != nil {
log.Error("could not start challenge container", "cause", err)
return fmt.Errorf("error starting container")
}
d.writeDeployChannel(deployKey, DeployStatus{Status: statusSuccess, Message: ``})
return nil
}
func (d *DockerDeployer) containerExistsForTeam(ctx context.Context, log *slog.Logger, job *DeployJob) error {
filters := client.Filters{}
filters.Add("label", ContainerLabelChallenge+"="+job.challenge)
filters.Add("label", ContainerLabelForTeam+"="+job.team)
containers, err := d.client.ContainerList(ctx, client.ContainerListOptions{
All: true,
Filters: filters,
})
if err != nil {
log.Error("could not pull list challenges", "cause", err)
return fmt.Errorf("error checking uniqueness")
}
if len(containers.Items) > 0 {
return fmt.Errorf("instance of challenge already deployed for team")
}
return nil
}
func (d *DockerDeployer) pullImage(ctx context.Context, log *slog.Logger, imageName string) error {
resp, err := d.client.ImagePull(ctx, imageName, client.ImagePullOptions{
RegistryAuth: d.registryLogin(),
})
if err != nil {
log.Error("could not pull challenge image", "cause", err)
return fmt.Errorf("error pulling challenge image")
}
defer resp.Close()
resp.Wait(ctx)
return nil
}
func (d *DockerDeployer) readImageMetadata(ctx context.Context, log *slog.Logger, imageName string) (*ImageConfig, error) {
inspect, err := d.client.ImageInspect(ctx, imageName)
if err != nil {
log.Error("could not inspect image", "cause", err)
return nil, fmt.Errorf("error loading challenge configuration")
}
imageCfg, err := extractImageConfig(inspect.Config.Labels)
if err != nil {
log.Error("invalid challenge configuration", "cause", err)
return nil, fmt.Errorf("error loading challenge configuration")
}
return &imageCfg, nil
}
func (d *DockerDeployer) setupNetwork(ctx context.Context, log *slog.Logger, job *DeployJob, expiry time.Time, routeName string, proxy bool) (string, error) {
resp, err := d.client.NetworkCreate(ctx, "n-"+routeName, client.NetworkCreateOptions{
Driver: "bridge",
Labels: map[string]string{
ContainerLabelManaged: "yes",
ContainerLabelCreatedAt: strconv.FormatInt(time.Now().Unix(), 10),
ContainerLabelExpiresAt: strconv.FormatInt(expiry.Unix(), 10),
ContainerLabelDeployKey: job.deployKey,
ContainerLabelChallenge: job.challenge,
ContainerLabelForTeam: job.team,
},
})
if err != nil {
log.Error("could not create challenge network", "cause", err)
return "", fmt.Errorf("could not create challenge network")
}
if proxy {
if _, err := d.client.NetworkConnect(ctx, resp.ID, client.NetworkConnectOptions{
Container: d.proxyContainerName,
}); err != nil {
log.Error("could not connect proxy to challenge network", "cause", err)
return "", fmt.Errorf("could not connect proxy to challenge network")
}
}
return resp.ID, nil
}
func (d *DockerDeployer) createContainer(ctx context.Context, log *slog.Logger, job *DeployJob, expiry time.Time, imageName, routeName, networkID, url string, imageCfg *ImageConfig) (string, error) {
now := time.Now()
labels := map[string]string{
ContainerLabelManaged: "yes",
ContainerLabelCreatedAt: strconv.FormatInt(now.Unix(), 10),
ContainerLabelExpiresAt: strconv.FormatInt(expiry.Unix(), 10),
ContainerLabelDeployKey: job.deployKey,
ContainerLabelChallenge: job.challenge,
ContainerLabelForTeam: job.team,
ContainerLabelAddress: url,
}
if imageCfg.ProxyEnable {
//TODO do tcp
maps.Copy(labels, map[string]string{
ContainerLabelAddressFormat: "https",
"traefik.enable": "true",
"traefik.docker.network": "n-" + routeName,
"traefik.http.routers." + routeName + ".entrypoints": "websecure",
"traefik.http.routers." + routeName + ".rule": "Host(`" + url + "`)",
"traefik.http.routers." + routeName + ".tls": "true",
"traefik.http.routers." + routeName + ".service": routeName,
"traefik.http.services." + routeName + ".loadbalancer.server.port": strconv.Itoa(imageCfg.Port),
})
}
resp, err := d.client.ContainerCreate(ctx, client.ContainerCreateOptions{
Image: imageName,
Config: &container.Config{
Labels: labels,
},
Name: "c-" + routeName,
HostConfig: &container.HostConfig{
Resources: container.Resources{
Memory: imageCfg.Limits.Memory,
NanoCPUs: imageCfg.Limits.CPU,
},
ReadonlyRootfs: imageCfg.Security.ReadOnlyFS,
SecurityOpt: imageCfg.Security.SecurityOpt,
CapAdd: imageCfg.Security.CapAdd,
CapDrop: imageCfg.Security.CapDrop,
},
NetworkingConfig: &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
networkID: {},
},
},
})
if err != nil {
log.Error("could not create challenge container", "cause", err)
return "", fmt.Errorf("error creating container")
}
return resp.ID, nil
}