diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 2b205324c..d9975e7e4 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -1,30 +1,39 @@ package onedrive_sharelink import ( + "bytes" "context" "encoding/json" "fmt" "io" "net/http" + "net/url" stdpath "path" + "regexp" "strings" "sync" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) const ( - headerTTL = 25 * time.Minute - directLinkTTL = 20 * time.Minute + headerTTL = 25 * time.Minute + driveTokenTTL = 20 * time.Minute + directLinkTTL = 20 * time.Minute + uploadSessionChunk = 10 * 1024 * 1024 ) type OnedriveSharelink struct { @@ -82,10 +91,17 @@ func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model. if err != nil { return nil, err } + folderSizes, err := d.driveChildrenFolderSizes(ctx, dir.GetPath()) + if err != nil { + log.Warnf("onedrive_sharelink: failed to get folder sizes for %s: %+v", dir.GetPath(), err) + } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { obj := fileToObj(src) + if size, ok := folderSizes[obj.GetName()]; ok { + obj.Size = size + } obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) return obj, nil }) @@ -125,8 +141,30 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model } func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - // TODO create folder, optional - return errs.NotImplement + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return err + } + apiURL := injectAccessToken(d.drivePathAPIURL(parentDir.GetPath())+"/children", token) + body := map[string]any{ + "name": dirName, + "folder": map[string]any{}, + "@microsoft.graph.conflictBehavior": "fail", + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + return nil + case http.StatusConflict: + return errs.ObjectAlreadyExists + default: + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create folder, status code: %d, body: %s", resp.StatusCode, string(data)) + } } func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -150,8 +188,422 @@ func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error { } func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO upload file, optional - return errs.NotImplement + info, err := d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), stream.GetName()), stream.GetSize()) + if err != nil { + return err + } + if info.ChunkSize == 0 { + return d.uploadContent(ctx, info.UploadURL, stream, up) + } + return d.uploadToSession(ctx, info.UploadURL, stream, up) +} + +func (d *OnedriveSharelink) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if d.DisableDiskUsage { + return nil, errs.NotImplement + } + size, err := d.driveItemSize(ctx, "/") + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: size, + UsedSpace: size, + }, + }, nil +} + +func (d *OnedriveSharelink) driveItemSize(ctx context.Context, path string) (int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return 0, err + } + apiURL := injectAccessToken(d.drivePathAPIURL(path), token) + resp, err := d.doJSON(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("failed to get folder details, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var item struct { + Size int64 `json:"size"` + } + if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { + return 0, err + } + return item.Size, nil +} + +func (d *OnedriveSharelink) driveChildrenFolderSizes(ctx context.Context, path string) (map[string]int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + rawURL := d.drivePathAPIURL(path) + "/children?$select=name,size,folder" + sizes := make(map[string]int64) + for rawURL != "" { + resp, err := d.doJSON(ctx, http.MethodGet, injectAccessToken(rawURL, token), nil) + if err != nil { + return nil, err + } + var data struct { + Value []struct { + Name string `json:"name"` + Size int64 `json:"size"` + Folder any `json:"folder"` + } `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + func() { + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + err = fmt.Errorf("failed to get children details, status code: %d, body: %s", resp.StatusCode, string(body)) + return + } + err = json.NewDecoder(resp.Body).Decode(&data) + }() + if err != nil { + return nil, err + } + for _, item := range data.Value { + if item.Folder != nil { + sizes[item.Name] = item.Size + } + } + rawURL = data.NextLink + } + return sizes, nil +} + +func (d *OnedriveSharelink) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveSharelink) GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + if tool != "HttpDirect" { + return nil, errs.NotImplement + } + return d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), fileName), fileSize) +} + +func (d *OnedriveSharelink) createUploadInfo(ctx context.Context, path string, fileSize int64) (*model.HttpDirectUploadInfo, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + if fileSize == 0 { + return &model.HttpDirectUploadInfo{ + UploadURL: injectAccessToken(d.drivePathAPIURL(path)+"/content", token), + Method: http.MethodPut, + }, nil + } + apiURL := injectAccessToken(d.drivePathAPIURL(path)+"/createUploadSession", token) + body := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to create upload session, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var data uploadSessionResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, err + } + if data.UploadURL == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: data.UploadURL, + ChunkSize: uploadSessionChunk, + Method: http.MethodPut, + }, nil +} + +func (d *OnedriveSharelink) uploadContent(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: file, + Size: file.GetSize(), + }, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader) + if err != nil { + return err + } + req.ContentLength = file.GetSize() + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to upload content, status code: %d, body: %s", resp.StatusCode, string(data)) + } + return nil +} + +func (d *OnedriveSharelink) uploadToSession(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + if file.GetSize() <= 0 { + return d.uploadSessionChunk(ctx, uploadURL, file, 0, 0, file.GetSize()) + } + ss, err := streamPkg.NewStreamSectionReader(file, uploadSessionChunk, &up) + if err != nil { + return err + } + var finish int64 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, int64(uploadSessionChunk)) + rd, err := ss.GetSectionReader(finish, byteSize) + if err != nil { + return err + } + err = retry.Do( + func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + return d.uploadSessionChunk(ctx, uploadURL, rd, finish, byteSize, file.GetSize()) + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + ss.FreeSectionReader(rd) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + } + return nil +} + +func (d *OnedriveSharelink) uploadSessionChunk(ctx context.Context, uploadURL string, reader io.Reader, start, size, total int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, driver.NewLimitedUploadStream(ctx, reader)) + if err != nil { + return err + } + req.ContentLength = size + if total > 0 { + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, start+size-1, total)) + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch { + case resp.StatusCode >= 500 && resp.StatusCode <= 504: + return fmt.Errorf("server error: %d", resp.StatusCode) + case resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted: + data, _ := io.ReadAll(resp.Body) + return errors.New(string(data)) + default: + return nil + } +} + +func (d *OnedriveSharelink) drivePathAPIURL(path string) string { + drivePath := stdpath.Join(d.driveRootPath, path) + drivePath = utils.FixAndCleanPath(drivePath) + if drivePath == "/" { + return d.DriveURL + "/root" + } + return fmt.Sprintf("%s/root:%s:", d.DriveURL, utils.EncodePath(drivePath, true)) +} + +func injectAccessToken(rawURL, token string) string { + if token == "" { + return rawURL + } + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + values := u.Query() + if strings.HasPrefix(token, "access_token=") { + values.Set("access_token", strings.TrimPrefix(token, "access_token=")) + } else { + values.Set("access_token", token) + } + u.RawQuery = values.Encode() + return u.String() +} + +func (d *OnedriveSharelink) doJSON(ctx context.Context, method, rawURL string, body any) (*http.Response, error) { + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, reader) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json;odata.metadata=minimal") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return base.HttpClient.Do(req) +} + +func (d *OnedriveSharelink) getValidDriveAccessToken(ctx context.Context) (string, error) { + d.headerMu.RLock() + token := d.DriveAccessToken + expired := time.Since(time.Unix(d.DriveTokenTime, 0)) > driveTokenTTL + d.headerMu.RUnlock() + if token != "" && !expired { + return token, nil + } + if err := d.refreshDriveContext(ctx); err != nil { + d.headerMu.RLock() + token = d.DriveAccessToken + d.headerMu.RUnlock() + if token != "" { + log.Warnf("onedrive_sharelink: use cached drive access token after refresh failure: %+v", err) + return token, nil + } + return "", err + } + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return d.DriveAccessToken, nil +} + +func (d *OnedriveSharelink) refreshDriveContext(ctx context.Context) error { + header, err := d.getValidHeaders(ctx) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.ShareLinkURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + redirectURL := resp.Header.Get("Location") + if redirectURL == "" && resp.Request != nil && resp.Request.URL != nil { + redirectURL = resp.Request.URL.String() + } + if redirectURL == "" { + return fmt.Errorf("share link did not return redirect URL") + } + return d.refreshDriveContextFromRedirect(ctx, redirectURL, header) +} + +func (d *OnedriveSharelink) refreshDriveContextFromRedirect(ctx context.Context, redirectURL string, header http.Header) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("onedrive page request failed, status code: %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + ctxInfo, err := parsePageContext(body) + if err != nil { + return err + } + rootPath, err := driveRootPathFromRedirect(redirectURL, ctxInfo.ListURL) + if err != nil { + return err + } + d.headerMu.Lock() + d.DriveURL = ctxInfo.DriveInfo.DriveURL + d.DriveAccessToken = ctxInfo.DriveInfo.DriveAccessToken + d.DriveTokenTime = time.Now().Unix() + d.driveRootPath = rootPath + d.headerMu.Unlock() + return nil +} + +func driveRootPathFromRedirect(redirectURL, listURL string) (string, error) { + u, err := url.Parse(redirectURL) + if err != nil { + return "", err + } + id := u.Query().Get("id") + if id == "" { + return "/", nil + } + if listURL == "" { + return "/", nil + } + if id == listURL { + return "/", nil + } + prefix := strings.TrimRight(listURL, "/") + "/" + if strings.HasPrefix(id, prefix) { + return utils.FixAndCleanPath(strings.TrimPrefix(id, strings.TrimRight(listURL, "/"))), nil + } + return "/", nil +} + +var pageContextRE = regexp.MustCompile(`(?s)var _spPageContextInfo=(\{.*?\});_spPageContextInfo`) + +func parsePageContext(body []byte) (*pageContextInfo, error) { + match := pageContextRE.FindSubmatch(body) + if len(match) < 2 { + return nil, fmt.Errorf("failed to find _spPageContextInfo") + } + var info pageContextInfo + if err := json.Unmarshal(match[1], &info); err != nil { + return nil, err + } + if info.DriveInfo.DriveURL == "" { + return nil, fmt.Errorf("failed to get drive URL from page context") + } + if info.DriveInfo.DriveAccessToken == "" { + return nil, fmt.Errorf("failed to get drive access token from page context") + } + return &info, nil } //func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/onedrive_sharelink/meta.go b/drivers/onedrive_sharelink/meta.go index 453f51e2a..1aed1b0ba 100644 --- a/drivers/onedrive_sharelink/meta.go +++ b/drivers/onedrive_sharelink/meta.go @@ -11,15 +11,21 @@ type Addition struct { driver.RootPath ShareLinkURL string `json:"url" required:"true"` ShareLinkPassword string `json:"password"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Allow uploading directly to OneDrive without going through OpenList"` IsSharepoint bool downloadLinkPrefix string Headers http.Header HeaderTime int64 + DriveURL string + DriveAccessToken string + DriveTokenTime int64 + driveRootPath string } var config = driver.Config{ Name: "Onedrive Sharelink", - NoUpload: true, + LocalSort: true, DefaultRoot: "/", } diff --git a/drivers/onedrive_sharelink/types.go b/drivers/onedrive_sharelink/types.go index 02b1f5750..4405324d8 100644 --- a/drivers/onedrive_sharelink/types.go +++ b/drivers/onedrive_sharelink/types.go @@ -38,6 +38,19 @@ type Object struct { ContentDownloadURL string } +type uploadSessionResp struct { + UploadURL string `json:"uploadUrl"` +} + +type pageContextInfo struct { + ListURL string `json:"listUrl"` + DriveInfo struct { + DriveURL string `json:".driveUrl"` + DriveAccessToken string `json:".driveAccessToken"` + DriveAccessTokenV2 string `json:".driveAccessTokenV21"` + } `json:"driveInfo"` +} + // fileToObj converts an Item to an Object. func fileToObj(f Item) *Object { // Convert Size from string to int64. diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go index d4cd4229a..13785939f 100644 --- a/drivers/onedrive_sharelink/util.go +++ b/drivers/onedrive_sharelink/util.go @@ -310,38 +310,25 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, json.NewDecoder(resp.Body).Decode(&graphqlReq) log.Debugln("graphqlReq:", graphqlReq) filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row - if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" { - nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) - log.Debugln("nextHref:", nextHref) - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) + nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + if nextHref != "" { + listViewXml := strings.Replace(graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml, `"`, `\"`, -1) + renderListDataAsStreamVar := strings.Replace( + `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}`, + "REPLACEME", + listViewXml, + -1, + ) - listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml - log.Debugln("listViewXml:", listViewXml) - renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}` - listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1) - renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1) - - graphqlReqNEW := GraphQLNEWRequest{} - postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref - req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) - req.Header = tempHeader - - resp, err := client.Do(req) - if err != nil { - d.Headers, err = d.getHeaders(ctx) - if err != nil { - return nil, err - } - return d.getFiles(ctx, path) - } - defer resp.Body.Close() - json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - for graphqlReqNEW.ListData.NextHref != "" { - graphqlReqNEW = GraphQLNEWRequest{} + for nextHref != "" { + nextHref = nextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" + nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) + log.Debugln("nextHref:", nextHref) + graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) req.Header = tempHeader + resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) @@ -352,13 +339,9 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, } defer resp.Body.Close() json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) filesData = append(filesData, graphqlReqNEW.ListData.Row...) + nextHref = graphqlReqNEW.ListData.NextHref } - filesData = append(filesData, graphqlReqNEW.ListData.Row...) - } else { - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) } return filesData, nil }