Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(digital_ocean): refactor digital ocean provider and task to conform to new interfaces #161

Merged
merged 50 commits into from
Jan 27, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
baee8f5
feat: rewrite the docker provider
Zygimantass Jan 3, 2025
753c6e3
wip
nadim-az Jan 12, 2025
56d3199
cleanup
nadim-az Jan 13, 2025
31b3e3f
fixes
nadim-az Jan 13, 2025
1ef6607
mo fixes mo problems
nadim-az Jan 13, 2025
1d29ffb
more cleanup after rebase
nadim-az Jan 13, 2025
10e6b34
more cleanup after rebase
nadim-az Jan 13, 2025
ea45999
genesis functions should use RunCommandWhileStopped
nadim-az Jan 14, 2025
d8dd503
runcommand should determine task state and how to run container accor…
nadim-az Jan 15, 2025
ee40457
harden task tests
nadim-az Jan 15, 2025
46ee0b2
harden provider tests
nadim-az Jan 15, 2025
c948b2c
fix
nadim-az Jan 16, 2025
b9acaca
move serializable fields to state
nadim-az Jan 16, 2025
d527c0b
lint
nadim-az Jan 16, 2025
99eaf18
docker lint
nadim-az Jan 16, 2025
eab0355
initialize missing fields after deserialization
nadim-az Jan 17, 2025
0e8104f
restore provider functionality
nadim-az Jan 17, 2025
d4a7ef2
remove docker changes
nadim-az Jan 20, 2025
d80df8e
pr feedback
nadim-az Jan 20, 2025
9cd4c3a
use zap fields in log statement
nadim-az Jan 20, 2025
5e34abe
reorder import
nadim-az Jan 20, 2025
bd3d7d8
clean code plz
nadim-az Jan 20, 2025
0845905
use getState everywhere
nadim-az Jan 21, 2025
a119818
fix: init provider state before using it in creating the firewall
Zygimantass Jan 21, 2025
d6c2a1e
droplet id in task states should be a string
nadim-az Jan 21, 2025
f1ab792
get public ip address through ifconfig instead of user input
nadim-az Jan 21, 2025
ac0c8dc
missing logger in restorechain
nadim-az Jan 21, 2025
9737a09
test: add e2e test
nadim-az Jan 21, 2025
c0459e0
refactor
nadim-az Jan 21, 2025
bfbb614
create -> restore -> create
nadim-az Jan 21, 2025
55384ee
bug fix
nadim-az Jan 21, 2025
ae78d38
test: e2e docker test
nadim-az Jan 19, 2025
3c5e94f
genesis functions should use RunCommandWhileStopped
nadim-az Jan 14, 2025
0990f41
move serializable fields to state
nadim-az Jan 16, 2025
9b89f56
chore: remove provider reference from task struct
nadim-az Jan 17, 2025
f3b70e3
fix bad rebase
nadim-az Jan 17, 2025
f2c49c1
lint
nadim-az Jan 17, 2025
763152b
fix tests
nadim-az Jan 17, 2025
93dcdeb
use network name
nadim-az Jan 19, 2025
1e13c94
add back AllocatedIPs to state
nadim-az Jan 19, 2025
c6668a1
bad rebase
nadim-az Jan 20, 2025
d70948e
refactor image pull logic
nadim-az Jan 20, 2025
9a5568b
use getState everywhere
nadim-az Jan 21, 2025
6b600c7
remove provider name field from task state
nadim-az Jan 21, 2025
29f1a58
add test for remove task
nadim-az Jan 21, 2025
d3a70f1
move docker client to provider/clients
nadim-az Jan 21, 2025
e96f184
removeTaskFunc moved to top-level provider package
nadim-az Jan 21, 2025
ec0afde
move tests to cosmos package
nadim-az Jan 22, 2025
d930bbc
upgrade to v3
nadim-az Jan 23, 2025
67a425c
bad rebase
nadim-az Jan 23, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions core/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ require (
github.com/go-rod/rod v0.114.6
github.com/golangci/golangci-lint v1.56.2
github.com/matoous/go-nanoid/v2 v2.1.0
github.com/opencontainers/image-spec v1.1.0-rc2
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.13.6
github.com/puzpuzpuz/xsync/v3 v3.4.0
github.com/spf13/afero v1.11.0
github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.26.0
Expand Down Expand Up @@ -199,7 +199,6 @@ require (
github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
Expand Down
2 changes: 0 additions & 2 deletions core/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -742,8 +742,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4=
github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
github.com/quasilyte/go-ruleguard v0.4.0 h1:DyM6r+TKL+xbKB4Nm7Afd1IQh9kEUKQs2pboWGKtvQo=
github.com/quasilyte/go-ruleguard v0.4.0/go.mod h1:Eu76Z/R8IXtViWUIHkE3p8gdH3/PKk1eh3YGfaEof10=
github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo=
Expand Down
194 changes: 194 additions & 0 deletions core/provider/clients/docker_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package clients

import (
"context"
"fmt"
"go.uber.org/zap"
"io"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
dockerclient "github.com/docker/docker/client"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)

// DockerClient is a unified interface for interacting with Docker
// It combines functionality needed by both the Docker and DigitalOcean providers
type DockerClient interface {
// Container Operations
ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error)
ContainerStart(ctx context.Context, container string, options container.StartOptions) error
ContainerStop(ctx context.Context, container string, options container.StopOptions) error
ContainerRemove(ctx context.Context, container string, options container.RemoveOptions) error
ContainerInspect(ctx context.Context, container string) (types.ContainerJSON, error)
ContainerList(ctx context.Context, options container.ListOptions) ([]types.Container, error)
ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error)

// Container Exec Operations
ContainerExecCreate(ctx context.Context, container string, config container.ExecOptions) (types.IDResponse, error)
ContainerExecAttach(ctx context.Context, execID string, config container.ExecStartOptions) (types.HijackedResponse, error)
ContainerExecInspect(ctx context.Context, execID string) (container.ExecInspect, error)

// Container File Operations
CopyToContainer(ctx context.Context, container, path string, content io.Reader, options container.CopyToContainerOptions) error
CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, container.PathStat, error)
ContainerLogs(ctx context.Context, container string, options container.LogsOptions) (io.ReadCloser, error)

// Image Operations
ImageInspectWithRaw(ctx context.Context, imageID string) (types.ImageInspect, []byte, error)
ImagePull(ctx context.Context, logger *zap.Logger, refStr string, options image.PullOptions) error

// Volume Operations
VolumeCreate(ctx context.Context, options volume.CreateOptions) (volume.Volume, error)
VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error)
VolumeList(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error)
VolumeRemove(ctx context.Context, volumeID string, force bool) error

// Network Operations
NetworkCreate(ctx context.Context, name string, options network.CreateOptions) (network.CreateResponse, error)
NetworkInspect(ctx context.Context, networkID string, options network.InspectOptions) (network.Inspect, error)
NetworkRemove(ctx context.Context, networkID string) error

// System Operations
Ping(ctx context.Context) (types.Ping, error)
Close() error
}

// defaultDockerClient is the default implementation of DockerClient interface
type defaultDockerClient struct {
client *dockerclient.Client
}

func NewDockerClient(host string) (DockerClient, error) {
// If host is empty, use default Docker socket
if host == "" {
client, err := dockerclient.NewClientWithOpts()
if err != nil {
return nil, err
}
return &defaultDockerClient{client: client}, nil
}

host = fmt.Sprintf("tcp://%s:2375", host)

client, err := dockerclient.NewClientWithOpts(dockerclient.WithHost(host))
if err != nil {
return nil, err
}
return &defaultDockerClient{client: client}, nil
}

func (d *defaultDockerClient) Ping(ctx context.Context) (types.Ping, error) {
return d.client.Ping(ctx)
}

func (d *defaultDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) {
return d.client.ImageInspectWithRaw(ctx, image)
}

func (d *defaultDockerClient) ImagePull(ctx context.Context, logger *zap.Logger, ref string, options image.PullOptions) error {
_, _, err := d.client.ImageInspectWithRaw(ctx, ref)
if err != nil {
logger.Info("pulling image", zap.String("image", ref))
resp, err := d.client.ImagePull(ctx, ref, options)
if err != nil {
return fmt.Errorf("failed to pull docker image: %w", err)
}

defer resp.Close()
// throw away the image pull stdout response
_, err = io.Copy(io.Discard, resp)
if err != nil {
return fmt.Errorf("failed to pull docker image: %w", err)
}
return nil
}
return nil
}

func (d *defaultDockerClient) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error) {
return d.client.ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, containerName)
}

func (d *defaultDockerClient) ContainerList(ctx context.Context, options container.ListOptions) ([]types.Container, error) {
return d.client.ContainerList(ctx, options)
}

func (d *defaultDockerClient) ContainerStart(ctx context.Context, containerID string, options container.StartOptions) error {
return d.client.ContainerStart(ctx, containerID, options)
}

func (d *defaultDockerClient) ContainerStop(ctx context.Context, containerID string, options container.StopOptions) error {
return d.client.ContainerStop(ctx, containerID, options)
}

func (d *defaultDockerClient) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) {
return d.client.ContainerInspect(ctx, containerID)
}

func (d *defaultDockerClient) ContainerExecCreate(ctx context.Context, container string, config container.ExecOptions) (types.IDResponse, error) {
return d.client.ContainerExecCreate(ctx, container, config)
}

func (d *defaultDockerClient) ContainerExecAttach(ctx context.Context, execID string, config container.ExecStartOptions) (types.HijackedResponse, error) {
return d.client.ContainerExecAttach(ctx, execID, config)
}

func (d *defaultDockerClient) ContainerExecInspect(ctx context.Context, execID string) (container.ExecInspect, error) {
return d.client.ContainerExecInspect(ctx, execID)
}

func (d *defaultDockerClient) ContainerRemove(ctx context.Context, containerID string, options container.RemoveOptions) error {
return d.client.ContainerRemove(ctx, containerID, options)
}

func (d *defaultDockerClient) ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
return d.client.ContainerWait(ctx, containerID, condition)
}

func (d *defaultDockerClient) ContainerLogs(ctx context.Context, container string, options container.LogsOptions) (io.ReadCloser, error) {
return d.client.ContainerLogs(ctx, container, options)
}

func (d *defaultDockerClient) CopyToContainer(ctx context.Context, container, path string, content io.Reader, options container.CopyToContainerOptions) error {
return d.client.CopyToContainer(ctx, container, path, content, options)
}

func (d *defaultDockerClient) CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, container.PathStat, error) {
return d.client.CopyFromContainer(ctx, container, srcPath)
}

func (d *defaultDockerClient) VolumeCreate(ctx context.Context, options volume.CreateOptions) (volume.Volume, error) {
return d.client.VolumeCreate(ctx, options)
}

func (d *defaultDockerClient) VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error) {
return d.client.VolumeInspect(ctx, volumeID)
}

func (d *defaultDockerClient) VolumeList(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error) {
return d.client.VolumeList(ctx, options)
}

func (d *defaultDockerClient) VolumeRemove(ctx context.Context, volumeID string, force bool) error {
return d.client.VolumeRemove(ctx, volumeID, force)
}

func (d *defaultDockerClient) NetworkCreate(ctx context.Context, name string, options network.CreateOptions) (network.CreateResponse, error) {
return d.client.NetworkCreate(ctx, name, options)
}

func (d *defaultDockerClient) NetworkInspect(ctx context.Context, networkID string, options network.InspectOptions) (network.Inspect, error) {
return d.client.NetworkInspect(ctx, networkID, options)
}

func (d *defaultDockerClient) NetworkRemove(ctx context.Context, networkID string) error {
return d.client.NetworkRemove(ctx, networkID)
}

func (d *defaultDockerClient) Close() error {
return d.client.Close()
}
128 changes: 128 additions & 0 deletions core/provider/digitalocean/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package digitalocean

import (
"context"
"fmt"

"github.com/digitalocean/godo"
)

// DoClient defines the interface for DigitalOcean API operations used by the provider
type DoClient interface {
nadim-az marked this conversation as resolved.
Show resolved Hide resolved
// Droplet operations
CreateDroplet(ctx context.Context, req *godo.DropletCreateRequest) (*godo.Droplet, error)
GetDroplet(ctx context.Context, dropletID int) (*godo.Droplet, error)
DeleteDropletByTag(ctx context.Context, tag string) error
DeleteDropletByID(ctx context.Context, id int) error

// Firewall operations
CreateFirewall(ctx context.Context, req *godo.FirewallRequest) (*godo.Firewall, error)
DeleteFirewall(ctx context.Context, firewallID string) error

// SSH Key operations
CreateKey(ctx context.Context, req *godo.KeyCreateRequest) (*godo.Key, error)
DeleteKeyByFingerprint(ctx context.Context, fingerprint string) error
GetKeyByFingerprint(ctx context.Context, fingerprint string) (*godo.Key, error)

// Tag operations
CreateTag(ctx context.Context, req *godo.TagCreateRequest) (*godo.Tag, error)
DeleteTag(ctx context.Context, tag string) error
}

// godoClient implements the DoClient interface using the actual godo.Client
type godoClient struct {
*godo.Client
}

func NewGodoClient(token string) DoClient {
return &godoClient{Client: godo.NewFromToken(token)}
}

func checkResponse(res *godo.Response, err error) error {
if err != nil {
return err
}

if res.StatusCode > 299 || res.StatusCode < 200 {
return fmt.Errorf("unexpected status code: %d", res.StatusCode)
}

return nil
}

// Droplet operations
func (c *godoClient) CreateDroplet(ctx context.Context, req *godo.DropletCreateRequest) (*godo.Droplet, error) {
droplet, res, err := c.Droplets.Create(ctx, req)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return droplet, nil
}

func (c *godoClient) GetDroplet(ctx context.Context, dropletID int) (*godo.Droplet, error) {
droplet, res, err := c.Droplets.Get(ctx, dropletID)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return droplet, nil
}

func (c *godoClient) DeleteDropletByTag(ctx context.Context, tag string) error {
res, err := c.Droplets.DeleteByTag(ctx, tag)
return checkResponse(res, err)
}

func (c *godoClient) DeleteDropletByID(ctx context.Context, id int) error {
res, err := c.Droplets.Delete(ctx, id)
return checkResponse(res, err)
}

// Firewall operations
func (c *godoClient) CreateFirewall(ctx context.Context, req *godo.FirewallRequest) (*godo.Firewall, error) {
firewall, res, err := c.Firewalls.Create(ctx, req)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return firewall, nil
}

func (c *godoClient) DeleteFirewall(ctx context.Context, firewallID string) error {
res, err := c.Firewalls.Delete(ctx, firewallID)
return checkResponse(res, err)
}

// SSH Key operations
func (c *godoClient) CreateKey(ctx context.Context, req *godo.KeyCreateRequest) (*godo.Key, error) {
key, res, err := c.Keys.Create(ctx, req)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return key, nil
}

func (c *godoClient) DeleteKeyByFingerprint(ctx context.Context, fingerprint string) error {
res, err := c.Keys.DeleteByFingerprint(ctx, fingerprint)
return checkResponse(res, err)
}

func (c *godoClient) GetKeyByFingerprint(ctx context.Context, fingerprint string) (*godo.Key, error) {
key, res, err := c.Keys.GetByFingerprint(ctx, fingerprint)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return key, nil
}

// Tag operations
func (c *godoClient) CreateTag(ctx context.Context, req *godo.TagCreateRequest) (*godo.Tag, error) {
tag, res, err := c.Tags.Create(ctx, req)
if err := checkResponse(res, err); err != nil {
return nil, err
}
return tag, nil
}

func (c *godoClient) DeleteTag(ctx context.Context, tag string) error {
res, err := c.Tags.Delete(ctx, tag)
return checkResponse(res, err)
}
Loading
Loading