Skip to content

Commit

Permalink
Catalaog upload feature (vmware#70)
Browse files Browse the repository at this point in the history 
* Added first working implementation for vapp template upload. Issue vmware/go-vcloud-director#50

* Small refactoring and added documentation.

Signed-off-by: Vaidotas Bauzys <[email protected]>
  • Loading branch information
@vbauzys
vbauzys authored Aug 28, 2018
1 parent 555c04c commit 93f521e
Show file tree
Hide file tree
Showing 2 changed files with 478 additions and 2 deletions.
374 changes: 372 additions & 2 deletions govcd/catalog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,22 @@
package govcd

import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"github.com/vmware/go-vcloud-director/types/v56"
"io"
"io/ioutil"
"log"
"math"
"net/http"
"net/url"

types "github.com/vmware/go-vcloud-director/types/v56"
"os"
"path"
"path/filepath"
"strconv"
"time"
)

type Catalog struct {
Expand All @@ -23,6 +35,15 @@ func NewCatalog(c *Client) *Catalog {
}
}

type Envelope struct {
File []struct {
HREF string `xml:"href,attr"`
ID string `xml:"id,attr"`
Size int `xml:"size,attr"`
ChunkSize int `xml:"chunkSize,attr"`
} `xml:"References>File"`
}

func (c *Catalog) FindCatalogItem(catalogitem string) (CatalogItem, error) {

for _, cis := range c.Catalog.CatalogItems {
Expand Down Expand Up @@ -55,3 +76,352 @@ func (c *Catalog) FindCatalogItem(catalogitem string) (CatalogItem, error) {

return CatalogItem{}, fmt.Errorf("can't find catalog item: %s", catalogitem)
}

/**
Uploads an ova file to a catalog.
This method only uploads bits to vCD spool area.
On a very high level the flow is as follows
1. Makes a POST call to vCD to create the catalog item (also creates a transfer folder in the spool area and as result will give a sparse catalog item resource XML).
2. Wait for the links to the transfer folder to appear in the resource representation of the catalog item.
3. Start uploading bits to the transfer folder
4. Wait on the import task to finish on vCD side -> task success = upload complete
*/
func (c *Catalog) UploadOvf(ovaFileName, itemName, description string, chunkSize int) error {

catalogItemUploadURL, err := findCatalogItemUploadLink(c)
if err != nil {
return err
}

vappTemplateUrl, err := createItemForUpload(c.c, catalogItemUploadURL, itemName, description)
if err != nil {
return err
}

vappTemplate, err := queryVappTemplate(c.c, vappTemplateUrl)
if err != nil {
return err
}

ovfUploadHref, err := getOvfUploadLink(vappTemplate)
if err != nil {
return err
}

filesAbsPaths, err := Unpack(ovaFileName)
if err != nil {
return err
}

var ovfFileDesc Envelope
var tempPath string

for _, filePath := range filesAbsPaths {
if filepath.Ext(filePath) == ".ovf" {
ovfFileDesc, err = uploadOvfDescription(c.c, filePath, ovfUploadHref)
tempPath, _ = filepath.Split(filePath)
if err != nil {
return err
}
break
}
}

vappTemplate, err = waitForTempUploadLinks(c.c, vappTemplateUrl)

err = uploadFiles(c.c, vappTemplate, &ovfFileDesc, tempPath, filesAbsPaths)
if err != nil {
return err
}

log.Printf("[TRACE] Upload finished \n")
return nil
}

func uploadFiles(client *Client, vappTemplate *types.VAppTemplate, ovfFileDesc *Envelope, tempPath string, filesAbsPaths []string) error {
for _, item := range vappTemplate.Files.File {
if item.BytesTransferred == 0 {
if ovfFileDesc.File[0].ChunkSize != 0 {
chunkFilePaths := getChunkedFilePaths(tempPath, ovfFileDesc.File[0].HREF, ovfFileDesc.File[0].Size, ovfFileDesc.File[0].ChunkSize)
err := uploadMultiPartFile(client, chunkFilePaths, item.Link[0].HREF, int64(ovfFileDesc.File[0].Size))
if err != nil {
return err
}
} else {
_, err := uploadFile(client, item.Link[0].HREF, findFilePath(filesAbsPaths, item.Name), 0, item.Size)
if err != nil {
return err
}
}
}
}
return nil
}

func uploadMultiPartFile(client *Client, filePaths []string, uploadHREF string, totalBytesToUpload int64) error {
log.Printf("[TRACE] Upload multi part file: %v\n, href: %s, size: %v", filePaths, uploadHREF, totalBytesToUpload)

var uploadedBytes int64

for i, filePath := range filePaths {
log.Printf("[TRACE] Uploading file: %v\n", i+1)
tempVar, err := uploadFile(client, uploadHREF, filePath, uploadedBytes, totalBytesToUpload)
if err != nil {
return err
}
uploadedBytes += tempVar
}
return nil
}

/** Function waits until vCD provides temporary file upload links. */
func waitForTempUploadLinks(client *Client, vappTemplateUrl *url.URL) (*types.VAppTemplate, error) {
var vAppTemplate *types.VAppTemplate
var err error
for {
log.Printf("[TRACE] Sleep... for 5 seconds.\n")
time.Sleep(time.Second * 5)
vAppTemplate, err = queryVappTemplate(client, vappTemplateUrl)
if err != nil {
return nil, err
}
if len(vAppTemplate.Files.File) > 1 {
log.Printf("[TRACE] upload link prepared.\n")
break
}
}
return vAppTemplate, nil
}

func getOvfUploadLink(vappTemplate *types.VAppTemplate) (*url.URL, error) {
log.Printf("[TRACE] Parsing ofv upload link: %#v\n", vappTemplate)

ovfUploadHref, err := url.ParseRequestURI(vappTemplate.Files.File[0].Link[0].HREF)
if err != nil {
return nil, err
}

return ovfUploadHref, nil
}

func queryVappTemplate(client *Client, vappTemplateUrl *url.URL) (*types.VAppTemplate, error) {
log.Printf("[TRACE] Qeurying vapp template: %s\n", vappTemplateUrl)
request := client.NewRequest(map[string]string{}, "GET", *vappTemplateUrl, nil)
response, err := client.Http.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()

body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}

vappTemplateParsed := &types.VAppTemplate{}

err = xml.Unmarshal(body, &vappTemplateParsed)
if err != nil {
return nil, err
}

log.Printf("[TRACE] Response: %v\n", response)
log.Printf("[TRACE] Response body: %s\n", string(body[:]))
log.Printf("[TRACE] Response body: %v\n", vappTemplateParsed)
return vappTemplateParsed, nil
}

/** Uploads ovf description file from unarchived provided ova file. As result vCD will generate temporary upload links which has to be queried later.
Function will return parsed part for upload files from description xml.*/
func uploadOvfDescription(client *Client, ovfFile string, ovfUploadUrl *url.URL) (Envelope, error) {
log.Printf("[TRACE] Uploding ovf description with file: %s and url: %s\n", ovfFile, ovfUploadUrl)
openedFile, err := os.Open(ovfFile)
if err != nil {
return Envelope{}, err
}

var buf bytes.Buffer
ovfReader := io.TeeReader(openedFile, &buf)

request := client.NewRequest(map[string]string{}, "PUT", *ovfUploadUrl, ovfReader)
request.Header.Add("Content-Type", "text/xml")

response, err := client.Http.Do(request)
if err != nil {
return Envelope{}, err
}

var ovfFileDesc Envelope
ovfXml, err := ioutil.ReadAll(&buf)
if err != nil {
return Envelope{}, err
}

err = xml.Unmarshal(ovfXml, &ovfFileDesc)
if err != nil {
return Envelope{}, err
}

openedFile.Close()
defer response.Body.Close()

body, err := ioutil.ReadAll(response.Body)
log.Printf("[TRACE] Response: %#v\n", response)
log.Printf("[TRACE] Response body: %s\n", string(body[:]))
log.Printf("[TRACE] Ovf file description file: %#v\n", ovfFileDesc)

return ovfFileDesc, nil
}

func findCatalogItemUploadLink(catalog *Catalog) (*url.URL, error) {
for _, item := range catalog.Catalog.Link {
if item.Type == "application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml" && item.Rel == "add" {
log.Printf("[TRACE] Found Catalong link for uplaod: %s\n", item.HREF)

uploadURL, err := url.ParseRequestURI(item.HREF)
if err != nil {
return nil, err
}

return uploadURL, nil
}
}
return nil, errors.New("catalog upload url isn't found")
}

func uploadFile(client *Client, uploadLink, filePath string, offset, fileSizeToUpload int64) (int64, error) {
log.Printf("[TRACE] Starting uploading: %s, offset: %v, fileze: %v, toLink: %s \n", filePath, offset, fileSizeToUpload, uploadLink)

file, err := os.Open(filePath)
if err != nil {
return 0, err
}

fileInfo, err := file.Stat()
if err != nil {
return 0, err
}

defer file.Close()

request, err := newFileUploadRequest(uploadLink, file, offset, fileInfo.Size(), fileSizeToUpload)
if err != nil {
return 0, err
}

response, err := client.Http.Do(request)
if err != nil {
return 0, err
}
defer response.Body.Close()

body, err := ioutil.ReadAll(response.Body)
if err != nil {
return 0, err
}
log.Printf("[TRACE] Response: %#v\n", response)
log.Printf("[TRACE] Response body: %s\n", string(body[:]))

if response.StatusCode != http.StatusOK {
return 0, errors.New("File " + filePath + " upload failed. Err: " + fmt.Sprintf("%#v", response) + " :: " + string(body[:]) + "\n")
}
return fileInfo.Size(), nil
}

func findFilePath(filesAbsPaths []string, fileName string) string {
for _, item := range filesAbsPaths {
_, file := filepath.Split(item)
if file == fileName {
return item
}
}
return ""
}

/** Initiates creation of item and returns ovf upload url for created item. */
func createItemForUpload(client *Client, createHREF *url.URL, catalogItemName string, itemDescription string) (*url.URL, error) {

reqBody := bytes.NewBufferString(
"<UploadVAppTemplateParams xmlns=\"http://www.vmware.com/vcloud/v1.5\" name=\"" + catalogItemName + "\" >" +
"<Description>" + itemDescription + "</Description>" +
"</UploadVAppTemplateParams>")

request := client.NewRequest(map[string]string{}, "POST", *createHREF, reqBody)
request.Header.Add("Content-Type", "application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml")

response, err := client.Http.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()

resBody, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}

// Unmarshal the XML.
catalogItemParsed := &types.CatalogItem{}
err = xml.Unmarshal(resBody, &catalogItemParsed)
if err != nil {
return nil, err
}

log.Printf("[TRACE] Response: %#v \n", response)
log.Printf("[TRACE] Response body: %s\n", string(resBody[:]))
log.Printf("[TRACE] Catalog item href to query vaapTemplate: %s\n", catalogItemParsed.Entity.HREF)

ovfUploadUrl, err := url.ParseRequestURI(catalogItemParsed.Entity.HREF)
if err != nil {
return nil, err
}

return ovfUploadUrl, nil
}

/** Create Request with right headers and range settings. Support multi part file upload. */
// TODO Creates a new file upload http request with optional extra params
func newFileUploadRequest(requestUrl string, file io.Reader, offset, fileSize, fileSizeToUpload int64) (*http.Request, error) {
log.Printf("[TRACE] Creating file upload request: %s, %v, %v, %v \n", requestUrl, offset, fileSize, fileSizeToUpload)

uploadReq, err := http.NewRequest("PUT", requestUrl, file)
if err != nil {
return nil, err
}

uploadReq.ContentLength = int64(fileSize)
uploadReq.Header.Set("Content-Length", strconv.FormatInt(uploadReq.ContentLength, 10))

rangeExpression := "bytes " + strconv.FormatInt(int64(offset), 10) + "-" + strconv.FormatInt(int64(offset+fileSize-1), 10) + "/" + strconv.FormatInt(int64(fileSizeToUpload), 10)
uploadReq.Header.Set("Content-Range", rangeExpression)

for key, value := range uploadReq.Header {
log.Printf("[TRACE] Header: %s :%s \n", key, value)
}

return uploadReq, nil
}

/** Helper method to get path to multi-part files.
For example a file called test.vmdk with total_file_size = 100 bytes and part_size = 40 bytes, implies the file is made of *3* part files.
- test.vmdk.000000000 = 40 bytes
- test.vmdk.000000001 = 40 bytes
- test.vmdk.000000002 = 20 bytes
Say base_dir = /dummy_path/, and base_file_name = test.vmdk then
the output of this function will be [/dummy_path/test.vmdk.000000000,
/dummy_path/test.vmdk.000000001, /dummy_path/test.vmdk.000000002]
*/
func getChunkedFilePaths(baseDir, baseFileName string, totalFileSize, partSize int) []string {
var filePaths []string
numbParts := math.Ceil(float64(totalFileSize) / float64(partSize))
for i := 0; i < int(numbParts); i++ {
temp := "000000000" + strconv.Itoa(i)
postfix := temp[len(temp)-9:]
filePath := path.Join(baseDir, baseFileName+"."+postfix)
filePaths = append(filePaths, filePath)
}

log.Printf("[TRACE] Chunked files file paths: %s \n", filePaths)
return filePaths
}
Loading

0 comments on commit 93f521e

Please sign in to comment.