Skip to content

Commit

Permalink
Merge pull request #204 from tiobe/33125-large_changed_files
Browse files Browse the repository at this point in the history 
Fixed `could not retrieve the changed files` on large pull requests
  • Loading branch information
@janssen-tiobe
janssen-tiobe authored Nov 7, 2023
2 parents 0fd300a + 0187eef commit 85fc3cf
Show file tree
Hide file tree
Showing 14 changed files with 546 additions and 168 deletions.
116 changes: 87 additions & 29 deletions dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ async function getChangedFilesOfCommit() {
// If a file is moved or renamed the status is 'renamed'.
if (item.status === 'renamed') {
// If a files has been moved without changes or if moved files are excluded, exclude them.
if (configuration_1.ticsConfig.excludeMovedFiles || item.changes === 0) {
if ((configuration_1.ticsConfig.excludeMovedFiles && item.changes === 0) || item.changes === 0) {
return false;
}
}
Expand Down Expand Up @@ -443,6 +443,7 @@ const canonical_path_1 = __nccwpck_require__(5806);
const logger_1 = __nccwpck_require__(6440);
const configuration_1 = __nccwpck_require__(5527);
const error_1 = __nccwpck_require__(5922);
const enums_1 = __nccwpck_require__(1655);
/**
* Sends a request to retrieve the changed files for a given pull request to the GitHub API.
* @returns List of changed files within the GitHub Pull request.
Expand All @@ -451,37 +452,67 @@ async function getChangedFilesOfPullRequest() {
const params = {
owner: configuration_1.githubConfig.owner,
repo: configuration_1.githubConfig.reponame,
pull_number: configuration_1.githubConfig.pullRequestNumber
pull_number: configuration_1.githubConfig.pullRequestNumber,
per_page: 100,
after: undefined
};
let response = [];
let files = [];
try {
logger_1.logger.header('Retrieving changed files.');
response = await configuration_1.octokit.paginate(configuration_1.octokit.rest.pulls.listFiles, params, response => {
let files = response.data
.filter(item => {
let response;
do {
response = await configuration_1.octokit.graphql(`query($owner: String!, $repo: String!, $pull_number: Int!, $per_page: Int!, $after: String) {
rateLimit {
remaining
}
repository(owner: $owner, name: $repo) {
pullRequest(number: $pull_number) {
files(first: $per_page, after: $after) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
path
changeType
additions
deletions
}
}
}
}
}`, params);
files = files.concat(response.repository.pullRequest.files.nodes
.map((item) => {
const changedFile = {
filename: (0, canonical_path_1.normalize)(item.path),
additions: item.additions,
deletions: item.deletions,
changes: item.additions + item.deletions,
status: enums_1.ChangeType[item.changeType]
};
return changedFile;
})
.filter((item) => {
if (item.status === 'renamed') {
// If a files has been moved without changes or if moved files are excluded, exclude them.
if (configuration_1.ticsConfig.excludeMovedFiles || item.changes === 0) {
if ((configuration_1.ticsConfig.excludeMovedFiles && item.changes === 0) || item.changes === 0) {
return false;
}
}
return true;
})
.map(item => {
// If a file is moved or renamed the status is 'renamed'.
item.filename = (0, canonical_path_1.normalize)(item.filename);
logger_1.logger.debug(item.filename);
return item;
});
return files ? files : [];
});
return true;
}));
params.after = response.repository.pullRequest.files.pageInfo.endCursor;
} while (response.repository.pullRequest.files.pageInfo.hasNextPage);
logger_1.logger.info('Retrieved changed files from pull request.');
}
catch (error) {
const message = (0, error_1.handleOctokitError)(error);
throw Error(`Could not retrieve the changed files: ${message}`);
}
return response;
return files;
}
exports.getChangedFilesOfPullRequest = getChangedFilesOfPullRequest;
/**
Expand Down Expand Up @@ -575,7 +606,7 @@ exports.postNothingAnalyzedReview = postNothingAnalyzedReview;
"use strict";

Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.Events = exports.Status = void 0;
exports.ChangeType = exports.Events = exports.Status = void 0;
var Status;
(function (Status) {
Status["FAILED"] = "FAILED";
Expand All @@ -589,6 +620,15 @@ var Events;
Events["COMMENT"] = "COMMENT";
Events["REQUEST_CHANGES"] = "REQUEST_CHANGES";
})(Events || (exports.Events = Events = {}));
var ChangeType;
(function (ChangeType) {
ChangeType["ADDED"] = "added";
ChangeType["CHANGED"] = "changed";
ChangeType["COPIED"] = "copied";
ChangeType["DELETED"] = "removed";
ChangeType["MODIFIED"] = "modified";
ChangeType["RENAMED"] = "renamed";
})(ChangeType || (exports.ChangeType = ChangeType = {}));


/***/ }),
Expand Down Expand Up @@ -918,19 +958,37 @@ function createReviewComments(annotations, changedFiles) {
let postable = [];
groupedAnnotations.forEach(annotation => {
const displayCount = annotation.count === 1 ? '' : `(${annotation.count}x) `;
if (annotation.diffLines?.includes(annotation.line)) {
logger_1.logger.debug(`Postable: ${JSON.stringify(annotation)}`);
postable.push({
title: `${annotation.instanceName}: ${annotation.rule}`,
body: createBody(annotation, displayCount),
path: annotation.path,
line: annotation.line
});
if (configuration_1.githubConfig.eventName === 'pull_request') {
if (changedFiles.find(c => annotation.fullPath.includes(c.filename))) {
logger_1.logger.debug(`Postable: ${JSON.stringify(annotation)}`);
postable.push({
title: `${annotation.instanceName}: ${annotation.rule}`,
body: createBody(annotation, displayCount),
path: annotation.path,
line: annotation.line
});
}
else {
annotation.displayCount = displayCount;
logger_1.logger.debug(`Unpostable: ${JSON.stringify(annotation)}`);
unpostable.push(annotation);
}
}
else {
annotation.displayCount = displayCount;
logger_1.logger.debug(`Unpostable: ${JSON.stringify(annotation)}`);
unpostable.push(annotation);
if (annotation.diffLines?.includes(annotation.line)) {
logger_1.logger.debug(`Postable: ${JSON.stringify(annotation)}`);
postable.push({
title: `${annotation.instanceName}: ${annotation.rule}`,
body: createBody(annotation, displayCount),
path: annotation.path,
line: annotation.line
});
}
else {
annotation.displayCount = displayCount;
logger_1.logger.debug(`Unpostable: ${JSON.stringify(annotation)}`);
unpostable.push(annotation);
}
}
});
logger_1.logger.info('Created review comments from annotations.');
Expand Down
2 changes: 1 addition & 1 deletion src/github/commits.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export async function getChangedFilesOfCommit(): Promise<ChangedFile[]> {
// If a file is moved or renamed the status is 'renamed'.
if (item.status === 'renamed') {
// If a files has been moved without changes or if moved files are excluded, exclude them.
if (ticsConfig.excludeMovedFiles || item.changes === 0) {
if ((ticsConfig.excludeMovedFiles && item.changes === 0) || item.changes === 0) {
return false;
}
}
Expand Down
48 changes: 43 additions & 5 deletions src/github/interfaces.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,15 +122,53 @@ export interface Comment {
}

export interface ChangedFile {
sha: string;
filename: string;
status: 'added' | 'removed' | 'modified' | 'renamed' | 'copied' | 'changed' | 'unchanged';
status: 'added' | 'changed' | 'copied' | 'removed' | 'modified' | 'renamed' | 'unchanged';
additions: number;
deletions: number;
changes: number;
blob_url: string;
raw_url: string;
contents_url: string;
sha?: string;
blob_url?: string;
raw_url?: string;
contents_url?: string;
patch?: string | undefined;
previous_filename?: string | undefined;
}

export type GraphQlParams = {
owner: string;
repo: string;
pull_number: number;
per_page: number;
after?: string;
};

export interface GraphQlChangedFile {
path: string;
changeType: 'ADDED' | 'CHANGED' | 'COPIED' | 'DELETED' | 'MODIFIED' | 'RENAMED';
additions: number;
deletions: number;
}

export interface GraphQlResponse<T> {
repository: {
pullRequest: T;
};
}

export interface ChangedFileResData {
files: {
totalCount: number;
pageInfo: {
endCursor: string;
hasNextPage: boolean;
};
nodes: GraphQlChangedFile[];
};
}

export interface HeadRepositoryOwnerResData {
headRepositoryOwner: {
login: string;
};
}
81 changes: 58 additions & 23 deletions src/github/pulls.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,83 @@ import { writeFileSync } from 'fs';
import { normalize, resolve } from 'canonical-path';
import { logger } from '../helper/logger';
import { githubConfig, octokit, ticsConfig } from '../configuration';
import { ChangedFile } from './interfaces';
import { ChangedFile, ChangedFileResData, GraphQlResponse, GraphQlChangedFile, GraphQlParams } from './interfaces';
import { handleOctokitError as getMessageFromOctokitError } from '../helper/error';
import { ChangeType } from '../helper/enums';

/**
* Sends a request to retrieve the changed files for a given pull request to the GitHub API.
* @returns List of changed files within the GitHub Pull request.
*/
export async function getChangedFilesOfPullRequest(): Promise<ChangedFile[]> {
const params = {
const params: GraphQlParams = {
owner: githubConfig.owner,
repo: githubConfig.reponame,
pull_number: githubConfig.pullRequestNumber
pull_number: githubConfig.pullRequestNumber,
per_page: 100,
after: undefined
};
let response: ChangedFile[] = [];
let files: ChangedFile[] = [];
try {
logger.header('Retrieving changed files.');
response = await octokit.paginate(octokit.rest.pulls.listFiles, params, response => {
let files = response.data
.filter(item => {
if (item.status === 'renamed') {
// If a files has been moved without changes or if moved files are excluded, exclude them.
if (ticsConfig.excludeMovedFiles || item.changes === 0) {
return false;
let response: GraphQlResponse<ChangedFileResData>;
do {
response = await octokit.graphql<GraphQlResponse<ChangedFileResData>>(
`query($owner: String!, $repo: String!, $pull_number: Int!, $per_page: Int!, $after: String) {
rateLimit {
remaining
}
repository(owner: $owner, name: $repo) {
pullRequest(number: $pull_number) {
files(first: $per_page, after: $after) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
path
changeType
additions
deletions
}
}
}
}
return true;
})
.map(item => {
// If a file is moved or renamed the status is 'renamed'.
item.filename = normalize(item.filename);
logger.debug(item.filename);
return item;
});

return files ? files : [];
});
}`,
params
);
files = files.concat(
response.repository.pullRequest.files.nodes
.map((item: GraphQlChangedFile) => {
const changedFile: ChangedFile = {
filename: normalize(item.path),
additions: item.additions,
deletions: item.deletions,
changes: item.additions + item.deletions,
status: ChangeType[item.changeType]
};
return changedFile;
})
.filter((item: ChangedFile) => {
if (item.status === 'renamed') {
// If a files has been moved without changes or if moved files are excluded, exclude them.
if ((ticsConfig.excludeMovedFiles && item.changes === 0) || item.changes === 0) {
return false;
}
}
logger.debug(item.filename);
return true;
})
);
params.after = response.repository.pullRequest.files.pageInfo.endCursor;
} while (response.repository.pullRequest.files.pageInfo.hasNextPage);
logger.info('Retrieved changed files from pull request.');
} catch (error: unknown) {
const message = getMessageFromOctokitError(error);
throw Error(`Could not retrieve the changed files: ${message}`);
}
return response;
return files;
}

/**
Expand Down
9 changes: 9 additions & 0 deletions src/helper/enums.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,12 @@ export enum Events {
'COMMENT' = 'COMMENT',
'REQUEST_CHANGES' = 'REQUEST_CHANGES'
}

export enum ChangeType {
ADDED = 'added',
CHANGED = 'changed',
COPIED = 'copied',
DELETED = 'removed',
MODIFIED = 'modified',
RENAMED = 'renamed'
}
Loading

0 comments on commit 85fc3cf

Please sign in to comment.