Skip to content

added file stream endpoint #232

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nandit123 wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

added file stream endpoint #232

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e53064f
added file stream endpoint
nandit123 Nov 3, 2020
c1ef0e7
added test case for add File Stream endpoint
nandit123 Nov 4, 2020
e6f0011
changed \ to fileath.Join() to join a path
nandit123 Nov 4, 2020
47bc26a
fixed changes, removed unnecessary code
nandit123 Nov 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions core/space/domain/domain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,12 @@ type AddItemsResponse struct {
Error error
}

type AddItemStreamResponse struct {
Result string
TotalBytes int64
Error error
Comment thread
dmerrill6 marked this conversation as resolved.
Outdated
}

type Member struct {
Address string `json:"address"`
PublicKey string `json:"publicKey"`
Expand Down
55 changes: 55 additions & 0 deletions core/space/services/services_fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,61 @@ func (s *Space) AddItems(ctx context.Context, sourcePaths []string, targetPath s
return results, totalsRes, nil
}

func (s *Space) AddItemStream(ctx context.Context, fileName string, targetPath string, bucketName string) (domain.AddItemStreamResponse, error) {
err := s.waitForTextileInit(ctx)
if err != nil {
return domain.AddItemStreamResponse{}, err
}
path, err := os.Getwd()
fileLocalPath := filepath.Join(path, fileName)
f, err := os.Open(fileLocalPath)
if err != nil {
log.Error(fmt.Sprintf("error opening path %s", fileLocalPath), err)
return domain.AddItemStreamResponse{}, err
}
defer closeFile(f, fileLocalPath)

var targetPathBucket string
if targetPath == "" || targetPath == "/" {
targetPathBucket = fileName
} else {
targetPathBucket = targetPath + "/" + fileName
}
b, err := s.getBucketWithFallback(ctx, bucketName)
if err != nil {
return domain.AddItemStreamResponse{}, err
}
_, _, err2 := b.UploadFile(ctx, targetPathBucket, f)
Comment thread
nandit123 marked this conversation as resolved.
Outdated
if err2 != nil {
log.Error(fmt.Sprintf("error creating targetPath %s in bucket %s", targetPathBucket, b.Key()), err2)
return domain.AddItemStreamResponse{}, err
}
fi, err := f.Stat()
var fileSize int64 = 0
if err == nil {
fileSize = fi.Size()
}
return domain.AddItemStreamResponse{
Result: "File Added",
TotalBytes: fileSize,
}, err
}

func closeFile(f *os.File, fileLocalPath string) {
fmt.Println("closing")
Comment thread
nandit123 marked this conversation as resolved.
Outdated
err := f.Close()

if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
Comment thread
nandit123 marked this conversation as resolved.
Outdated
os.Exit(1)
} else {
e := os.Remove(fileLocalPath)
if e != nil {
log.Error(fmt.Sprintf("File remove error: "), e)
}
}
}

// AddItemWithReader uploads content of the reader to the targetPath on the bucket specified
//
// Note: the AddItemResult returns an empty SourcePath
Expand Down
1 change: 1 addition & 0 deletions core/space/space.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Service interface {
CreateBucket(ctx context.Context, slug string) (textile.Bucket, error)
ListBuckets(ctx context.Context) ([]textile.Bucket, error)
AddItems(ctx context.Context, sourcePaths []string, targetPath string, bucketName string) (<-chan domain.AddItemResult, domain.AddItemsResponse, error)
AddItemStream(ctx context.Context, fileName string, targetPath string, bucketName string) (domain.AddItemStreamResponse, error)
AddItemWithReader(ctx context.Context, reader io.Reader, targetPath, bucketName string) (domain.AddItemResult, error)
CreateIdentity(ctx context.Context, username string) (*domain.Identity, error)
GetIdentityByUsername(ctx context.Context, username string) (*domain.Identity, error)
Expand Down
44 changes: 44 additions & 0 deletions core/space/space_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,50 @@ func TestService_AddItems_OnError(t *testing.T) {
mockBucket.AssertNumberOfCalls(t, "UploadFile", len(getTempDir().fileNames))
}

func TestService_AddItemStream(t *testing.T) {
sv, _, tearDown := initTestService(t)

// setup tests
testKey := "bucketKey"
bucketPath := "/tests"

textileClient.On("GetDefaultBucket", mock.Anything).Return(mockBucket, nil)
textileClient.On("IsInitialized").Return(true)

mockBucket.On(
"Key",
).Return(testKey)

mockBucket.On(
"Slug",
).Return("personal")

mockPath.On("String").Return("hash")

fileName := "textFile.txt"
tmpFile, err := os.Create(fileName)

defer closeAndDelete(tmpFile)

defer tearDown()
mockBucket.On(
"UploadFile",
mock.Anything,
bucketPath+"/"+fileName,
mock.Anything,
).Return(nil, mockPath, nil)

res, err := sv.AddItemStream(context.Background(), fileName, bucketPath, "")

assert.Nil(t, err)
assert.NotEmpty(t, res)
assert.Equal(t, "File Added", res.Result)

// assert mocks
textileClient.AssertExpectations(t)
mockBucket.AssertNumberOfCalls(t, "UploadFile", 1)
}

func TestService_CreateIdentity(t *testing.T) {
sv, _, tearDown := initTestService(t)
defer tearDown()
Expand Down
12 changes: 6 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/fatih/color v1.9.0 // indirect
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/golang/protobuf v1.4.2
github.com/google/uuid v1.1.1
github.com/grpc-ecosystem/grpc-gateway v1.14.6
github.com/golang/protobuf v1.4.3
github.com/google/uuid v1.1.2
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/improbable-eng/grpc-web v0.13.0
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-ipfs v0.6.1-0.20200817102359-90a573354af2
Expand Down Expand Up @@ -53,11 +53,11 @@ require (
github.com/tyler-smith/go-bip39 v1.0.2
go.etcd.io/etcd v3.3.22+incompatible
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/net v0.0.0-20200707034311-ab3426394381
golang.org/x/net v0.0.0-20200822124328-c89045814202
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980
google.golang.org/genproto v0.0.0-20200702021140-07506425bd67
google.golang.org/grpc v1.31.0
google.golang.org/genproto v0.0.0-20201103154000-415bd0cd5df6
google.golang.org/grpc v1.33.1
google.golang.org/protobuf v1.25.0
gopkg.in/yaml.v2 v2.3.0 // indirect
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919
Expand Down
13 changes: 13 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
Expand Down Expand Up @@ -569,6 +571,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
Expand Down Expand Up @@ -608,6 +612,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o=
github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
Expand Down Expand Up @@ -1455,6 +1461,7 @@ github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3
github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
Expand Down Expand Up @@ -2261,6 +2268,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrS
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -2504,6 +2513,8 @@ google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200702021140-07506425bd67 h1:4BC1C1i30F3MZeiIO6y6IIo4DxrtOwITK87bQl3lhFA=
google.golang.org/genproto v0.0.0-20200702021140-07506425bd67/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201103154000-415bd0cd5df6 h1:rMoZiLTOobSD3eg30lPMcFkBFNSyKUQQIQlw/hsAXME=
google.golang.org/genproto v0.0.0-20201103154000-415bd0cd5df6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
Expand All @@ -2528,6 +2539,8 @@ google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200617041141-9a465503579e/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/grpc/examples v0.0.0-20200819190100-f640ae6a4f43/go.mod h1:wQWkdCkP0Pl3MzFPvfqTNUnXA2eIVY4eakDiKJvniKc=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
Expand Down
20 changes: 20 additions & 0 deletions grpc/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package grpc
import (
"context"
"errors"
"io/ioutil"

"github.com/FleekHQ/space-daemon/core/events"
"github.com/FleekHQ/space-daemon/core/space/domain"
Expand Down Expand Up @@ -241,6 +242,25 @@ func (srv *grpcServer) AddItems(request *pb.AddItemsRequest, stream pb.SpaceApi_
return nil
}

func (srv *grpcServer) AddItemStream(request *pb.AddItemStreamRequest, stream pb.SpaceApi_AddItemStreamServer) error {
ctx := stream.Context()

byteArray := request.Data
err := ioutil.WriteFile(request.FileName, byteArray, 0644) // filename also contain the file-extension
if err != nil {
// handle error
return err
}
result, err := srv.sv.AddItemStream(ctx, request.FileName, request.TargetPath, request.Bucket)
var r *pb.AddItemStreamResponse
r = &pb.AddItemStreamResponse{
TotalBytes: result.TotalBytes,
Result: result.Result,
}
stream.Send(r)
return nil
}

func (srv *grpcServer) CreateFolder(ctx context.Context, request *pb.CreateFolderRequest) (*pb.CreateFolderResponse, error) {
err := srv.sv.CreateFolder(ctx, request.Path, request.Bucket)
if err != nil {
Expand Down
Loading