From c455fbe5275b7ce0a59404671adc94c0ff494cbe Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 19 Jan 2021 00:44:37 +0000 Subject: [PATCH 1/2] OData pagination handling for GET requests using base client --- base/get.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++--- base/odata.go | 16 +++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 base/odata.go diff --git a/base/get.go b/base/get.go index 6ebb3541..fb74f576 100644 --- a/base/get.go +++ b/base/get.go @@ -1,9 +1,13 @@ package base import ( + "bytes" "context" + "encoding/json" "fmt" + "io/ioutil" "net/http" + "strings" ) // GetHttpRequestInput configures a GET request. @@ -11,6 +15,7 @@ type GetHttpRequestInput struct { ValidStatusCodes []int ValidStatusFunc ValidStatusFunc Uri Uri + rawUri string } // GetValidStatusCodes returns a []int of status codes considered valid for a GET request. @@ -26,17 +31,81 @@ func (i GetHttpRequestInput) GetValidStatusFunc() ValidStatusFunc { // Get performs a GET request. func (c Client) Get(ctx context.Context, input GetHttpRequestInput) (*http.Response, int, error) { var status int - url, err := c.buildUri(input.Uri) - if err != nil { - return nil, status, fmt.Errorf("unable to make request: %v", err) + + // Check for a raw uri, else build one from the Uri field + url := input.rawUri + if url == "" { + var err error + url, err = c.buildUri(input.Uri) + if err != nil { + return nil, status, fmt.Errorf("unable to make request: %v", err) + } } + + // Build a new request req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return nil, status, err } + + // Perform the request resp, status, err := c.performRequest(req, input) if err != nil { return nil, status, err } + + // Check for json content before handling pagination + contentType := strings.ToLower(resp.Header.Get("Content-Type")) + if strings.HasPrefix(contentType, "application/json") { + // Read the response body and close it + respBody, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + + // Unmarshall odata + var odata OData + if err := json.Unmarshal(respBody, &odata); err != nil { + return nil, status, err + } + + if odata.NextLink == nil || odata.Value == nil { + // No more pages, reassign response body and return + resp.Body = ioutil.NopCloser(bytes.NewBuffer(respBody)) + return resp, status, nil + } + + // Get the next page, recursively + nextInput := input + nextInput.rawUri = *odata.NextLink + nextResp, status, err := c.Get(ctx, nextInput) + if err != nil { + return resp, status, err + } + + // Read the next page response body and close it + nextRespBody, _ := ioutil.ReadAll(nextResp.Body) + nextResp.Body.Close() + + // Unmarshall odata from the next page + var nextOdata OData + if err := json.Unmarshal(nextRespBody, &nextOdata); err != nil { + return resp, status, err + } + + if nextOdata.Value != nil { + // Next page has results, append to current page + value := append(*odata.Value, *nextOdata.Value...) + nextOdata.Value = &value + } + + // Marshal the entire result, along with fields from the final page + newJson, err := json.Marshal(nextOdata) + if err != nil { + return resp, status, err + } + + // Reassign the response body + resp.Body = ioutil.NopCloser(bytes.NewBuffer(newJson)) + } + return resp, status, nil } diff --git a/base/odata.go b/base/odata.go new file mode 100644 index 00000000..a5e6b628 --- /dev/null +++ b/base/odata.go @@ -0,0 +1,16 @@ +package base + +import "encoding/json" + +type OData struct { + Context *string `json:"@odata.context"` + MetadataEtag *string `json:"@odata.metadataEtag"` + Type *string `json:"@odata.type"` + Count *string `json:"@odata.count"` + NextLink *string `json:"@odata.nextLink"` + Delta *string `json:"@odata.delta"` + DeltaLink *string `json:"@odata.deltaLink"` + Id *string `json:"@odata.id"` + Etag *string `json:"@odata.etag"` + Value *[]json.RawMessage `json:"value"` +} From f7fa82b7694a59a2b65bc67cb520b9272b17e378 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 19 Jan 2021 00:44:55 +0000 Subject: [PATCH 2/2] Fix up some test messages --- clients/applications_test.go | 4 ++-- clients/groups_test.go | 4 ++-- clients/serviceprincipals_test.go | 4 ++-- clients/users_test.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/clients/applications_test.go b/clients/applications_test.go index 324cd1d9..e1f2db7e 100644 --- a/clients/applications_test.go +++ b/clients/applications_test.go @@ -79,10 +79,10 @@ func testApplicationsClient_List(t *testing.T, c ApplicationsClientTest) (applic func testApplicationsClient_Get(t *testing.T, c ApplicationsClientTest, id string) (application *models.Application) { application, status, err := c.client.Get(c.connection.Context, id) if err != nil { - t.Fatalf("ApplicationsClient.Delete(): %v", err) + t.Fatalf("ApplicationsClient.Get(): %v", err) } if status < 200 || status >= 300 { - t.Fatalf("ApplicationsClient.Delete(): invalid status: %d", status) + t.Fatalf("ApplicationsClient.Get(): invalid status: %d", status) } if application == nil { t.Fatal("ApplicationsClient.Get(): application was nil") diff --git a/clients/groups_test.go b/clients/groups_test.go index 51e1a35b..862bd85d 100644 --- a/clients/groups_test.go +++ b/clients/groups_test.go @@ -123,10 +123,10 @@ func testGroupsClient_List(t *testing.T, c GroupsClientTest) (groups *[]models.G func testGroupsClient_Get(t *testing.T, c GroupsClientTest, id string) (group *models.Group) { group, status, err := c.client.Get(c.connection.Context, id) if err != nil { - t.Fatalf("GroupsClient.Delete(): %v", err) + t.Fatalf("GroupsClient.Get(): %v", err) } if status < 200 || status >= 300 { - t.Fatalf("GroupsClient.Delete(): invalid status: %d", status) + t.Fatalf("GroupsClient.Get(): invalid status: %d", status) } if group == nil { t.Fatal("GroupsClient.Get(): group was nil") diff --git a/clients/serviceprincipals_test.go b/clients/serviceprincipals_test.go index c674cad6..accc364b 100644 --- a/clients/serviceprincipals_test.go +++ b/clients/serviceprincipals_test.go @@ -90,10 +90,10 @@ func testServicePrincipalsClient_List(t *testing.T, c ServicePrincipalsClientTes func testServicePrincipalsClient_Get(t *testing.T, c ServicePrincipalsClientTest, id string) (servicePrincipal *models.ServicePrincipal) { servicePrincipal, status, err := c.client.Get(c.connection.Context, id) if err != nil { - t.Fatalf("ServicePrincipalsClient.Delete(): %v", err) + t.Fatalf("ServicePrincipalsClient.Get(): %v", err) } if status < 200 || status >= 300 { - t.Fatalf("ServicePrincipalsClient.Delete(): invalid status: %d", status) + t.Fatalf("ServicePrincipalsClient.Get(): invalid status: %d", status) } if servicePrincipal == nil { t.Fatal("ServicePrincipalsClient.Get(): servicePrincipal was nil") diff --git a/clients/users_test.go b/clients/users_test.go index 82f35e27..23943c24 100644 --- a/clients/users_test.go +++ b/clients/users_test.go @@ -80,10 +80,10 @@ func testUsersClient_List(t *testing.T, c UsersClientTest) (users *[]models.User func testUsersClient_Get(t *testing.T, c UsersClientTest, id string) (user *models.User) { user, status, err := c.client.Get(c.connection.Context, id) if err != nil { - t.Fatalf("UsersClient.Delete(): %v", err) + t.Fatalf("UsersClient.Get(): %v", err) } if status < 200 || status >= 300 { - t.Fatalf("UsersClient.Delete(): invalid status: %d", status) + t.Fatalf("UsersClient.Get(): invalid status: %d", status) } if user == nil { t.Fatal("UsersClient.Get(): user was nil")