Add rustfs #65
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| # 빌드 및 실행 단계 | ||
| FROM golang:1.24 | ||
|
|
||
| # 필수 패키지 설치 (libvirt 개발 패키지 포함) | ||
| RUN apt-get update && apt-get install -y \ | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "net/http" | ||
|
|
||
| "github.com/easy-cloud-Knet/KWS_Control/service" | ||
| "github.com/easy-cloud-Knet/KWS_Control/structure" | ||
| "github.com/easy-cloud-Knet/KWS_Control/util" | ||
| ) | ||
|
|
||
| type ApiTakeSnapshotRequest struct { | ||
| UUID structure.UUID `json:"uuid"` | ||
| SnapName string `json:"snapName"` | ||
| } | ||
|
|
||
| type ApiDeleteSnapshotRequest struct { | ||
| UUID structure.UUID `json:"uuid"` | ||
| SnapKey string `json:"snapKey"` | ||
| } | ||
|
|
||
| func (c *handlerContext) takeSnapshot(w http.ResponseWriter, r *http.Request) { | ||
| log := util.GetLogger() | ||
| defer r.Body.Close() | ||
|
|
||
| var req ApiTakeSnapshotRequest | ||
| if err := json.NewDecoder(r.Body).Decode(&req); err != nil { | ||
| util.RespondError(w, http.StatusBadRequest, "invalid request body") | ||
| return | ||
| } | ||
| if req.UUID == "" || req.SnapName == "" { | ||
| util.RespondError(w, http.StatusBadRequest, "uuid and snapName are required") | ||
| return | ||
| } | ||
|
|
||
| if err := service.TakeSnapshot(req.UUID, req.SnapName, c.context); err != nil { | ||
| log.Error("takeSnapshot: %v", err, true) | ||
| util.RespondError(w, http.StatusInternalServerError, err.Error()) | ||
| return | ||
| } | ||
|
|
||
| util.RespondJSON(w, http.StatusOK, map[string]string{"message": "snapshot taken"}) | ||
| } | ||
|
|
||
| func (c *handlerContext) listSnapshots(w http.ResponseWriter, r *http.Request) { | ||
| log := util.GetLogger() | ||
|
|
||
| uuid := structure.UUID(r.URL.Query().Get("uuid")) | ||
| if uuid == "" { | ||
| util.RespondError(w, http.StatusBadRequest, "uuid query parameter is required") | ||
| return | ||
| } | ||
|
|
||
| keys, err := service.ListSnapshots(uuid) | ||
| if err != nil { | ||
| log.Error("listSnapshots: %v", err, true) | ||
| util.RespondError(w, http.StatusInternalServerError, err.Error()) | ||
| return | ||
| } | ||
|
|
||
| util.RespondJSON(w, http.StatusOK, map[string][]string{"snapshots": keys}) | ||
| } | ||
|
|
||
| func (c *handlerContext) deleteSnapshot(w http.ResponseWriter, r *http.Request) { | ||
| log := util.GetLogger() | ||
| defer r.Body.Close() | ||
|
|
||
| var req ApiDeleteSnapshotRequest | ||
| if err := json.NewDecoder(r.Body).Decode(&req); err != nil { | ||
| util.RespondError(w, http.StatusBadRequest, "invalid request body") | ||
| return | ||
| } | ||
| if req.UUID == "" || req.SnapKey == "" { | ||
| util.RespondError(w, http.StatusBadRequest, "uuid and snapKey are required") | ||
| return | ||
| } | ||
|
|
||
| if err := service.DeleteSnapshot(req.UUID, req.SnapKey); err != nil { | ||
| log.Error("deleteSnapshot: %v", err, true) | ||
| util.RespondError(w, http.StatusInternalServerError, err.Error()) | ||
| return | ||
| } | ||
|
|
||
| w.WriteHeader(http.StatusOK) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package model | ||
|
|
||
| import "github.com/easy-cloud-Knet/KWS_Control/structure" | ||
|
|
||
| // TakeSnapshotRequest asks Core to take an external snapshot and upload to RustFS via presigned PUT URL. | ||
| type TakeSnapshotRequest struct { | ||
| UUID structure.UUID `json:"uuid"` | ||
| SnapKey string `json:"snapKey"` | ||
| PresignedURL string `json:"presignedUrl,omitempty"` | ||
| } | ||
|
|
||
| type TakeSnapshotResponse struct { | ||
| Message string `json:"message,omitempty"` | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| package client | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/config" | ||
| "github.com/aws/aws-sdk-go-v2/credentials" | ||
| "github.com/aws/aws-sdk-go-v2/service/s3" | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| "github.com/aws/aws-sdk-go-v2/service/s3/types" | ||
| ) | ||
|
|
||
| type RustFSClient struct { | ||
| client *s3.Client | ||
| } | ||
|
|
||
| var ( | ||
| rustFSInstance *RustFSClient | ||
| rustFSOnce sync.Once | ||
| rustFSErr error | ||
| ) | ||
|
|
||
| func GetRustFSClient() (*RustFSClient, error) { | ||
| rustFSOnce.Do(func() { | ||
| rustFSInstance, rustFSErr = NewRustFSClient() | ||
| }) | ||
| return rustFSInstance, rustFSErr | ||
| } | ||
|
|
||
| // Currently RustFSClient has solid s3.Client as its field, | ||
| // when logic get complex and needs for mocking arise, replace it with interface | ||
|
|
||
| func NewRustFSClient() (*RustFSClient, error) { | ||
| endpoint := os.Getenv("RUSTFS_ENDPOINT") | ||
| if endpoint == "" { | ||
| endpoint = "http://localhost:9000" | ||
| } | ||
|
Contributor
Author
There was a problem hiding this comment. minioadmin<-- docker-compose.test.yaml 파일의 인자를 일단 덮어쓰도록 했습니다.
kwonkwonn marked this conversation as resolved.
|
||
| accessKey := os.Getenv("RUSTFS_ACCESS_KEY") | ||
| if accessKey == "" { | ||
| accessKey = "minioadmin" | ||
| } | ||
| secretKey := os.Getenv("RUSTFS_SECRET_KEY") | ||
| if secretKey == "" { | ||
| secretKey = "minioadmin" | ||
| } | ||
| //TODO: replace unstable accesskey override when testing with proper method | ||
|
|
||
| cfg, err := config.LoadDefaultConfig(context.Background(), | ||
| config.WithRegion("us-east-1"), | ||
| config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| client := s3.NewFromConfig(cfg, func(o *s3.Options) { | ||
| o.BaseEndpoint = aws.String(endpoint) | ||
| o.UsePathStyle = true | ||
| }) | ||
|
|
||
| return &RustFSClient{client: client}, nil | ||
| } | ||
|
|
||
| func (c *RustFSClient) ListBuckets(ctx context.Context) ([]string, error) { | ||
| out, err := c.client.ListBuckets(ctx, &s3.ListBucketsInput{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| names := make([]string, len(out.Buckets)) | ||
| for i, b := range out.Buckets { | ||
| names[i] = aws.ToString(b.Name) | ||
| } | ||
| return names, nil | ||
| } | ||
|
|
||
| func (c *RustFSClient) CreateBucket(ctx context.Context, bucket string) error { | ||
| _, err := c.client.CreateBucket(ctx, &s3.CreateBucketInput{ | ||
| Bucket: aws.String(bucket), | ||
| }) | ||
| return err | ||
| } | ||
|
|
||
| // Refer: https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#NewPresignClient | ||
| // Direct listing, generating bucket and presigned URL is the role for RustFSClient(control) | ||
| // This Presigned url should be passed to Core client, and core should use this url to upload/download file. | ||
| func (c *RustFSClient) PresignPutObject(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { | ||
| presignClient := s3.NewPresignClient(c.client) | ||
| req, err := presignClient.PresignPutObject(ctx, &s3.PutObjectInput{ | ||
| Bucket: aws.String(bucket), | ||
| Key: aws.String(key), | ||
| }, s3.WithPresignExpires(expires)) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return req.URL, nil | ||
| } | ||
|
|
||
| func (c *RustFSClient) PresignGetObject(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { | ||
| presignClient := s3.NewPresignClient(c.client) | ||
| req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ | ||
| Bucket: aws.String(bucket), | ||
| Key: aws.String(key), | ||
| }, s3.WithPresignExpires(expires)) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return req.URL, nil | ||
| } | ||
|
|
||
| // HeadObject returns (true, nil) if the object exists, (false, nil) if only the object is missing, | ||
| // and (false, err) for all other errors including NoSuchBucket. | ||
| func (c *RustFSClient) HeadObject(ctx context.Context, bucket, key string) (bool, error) { | ||
| _, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ | ||
| Bucket: aws.String(bucket), | ||
| Key: aws.String(key), | ||
| }) | ||
| if err != nil { | ||
| var notFound *types.NotFound | ||
| if errors.As(err, ¬Found) { | ||
| return false, nil | ||
| } | ||
| return false, fmt.Errorf("HeadObject %s/%s: %w", bucket, key, err) | ||
| } | ||
| return true, nil | ||
| } | ||
|
|
||
| // ListObjects returns all object keys in the bucket matching the prefix. | ||
| // Returns an empty slice when no objects match. Returns an error if the bucket does not exist. | ||
| func (c *RustFSClient) ListObjects(ctx context.Context, bucket, prefix string) ([]string, error) { | ||
| out, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ | ||
| Bucket: aws.String(bucket), | ||
| Prefix: aws.String(prefix), | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("ListObjects %s: %w", bucket, err) | ||
| } | ||
| keys := make([]string, len(out.Contents)) | ||
| for i, obj := range out.Contents { | ||
| keys[i] = aws.ToString(obj.Key) | ||
| } | ||
| return keys, nil | ||
| } | ||
|
kwonkwonn marked this conversation as resolved.
|
||
|
|
||
| func (c *RustFSClient) DeleteObject(ctx context.Context, bucket, key string) error { | ||
| _, err := c.client.DeleteObject(ctx, &s3.DeleteObjectInput{ | ||
| Bucket: aws.String(bucket), | ||
| Key: aws.String(key), | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("DeleteObject %s/%s: %w", bucket, key, err) | ||
| } | ||
| return nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.