Skip to content

Commit

Permalink
Add test for Ground truth Job list (#7490)
Browse files Browse the repository at this point in the history
  • Loading branch information
@klakhov
klakhov authored Feb 22, 2024
1 parent 7286d65 commit 2c5bb69
Showing 1 changed file with 160 additions and 61 deletions.
221 changes: 160 additions & 61 deletions tests/cypress/e2e/features/ground_truth_jobs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,6 @@ context('Ground truth jobs', () => {
const taskName = `Annotation task for Case ${caseId}`;
const attrName = `Attr for Case ${caseId}`;
const textDefaultValue = 'Some default value for type Text';
const imagesCount = 10;
const imageFileName = 'ground_truth_1';
const width = 800;
const height = 800;
const posX = 10;
const posY = 10;
const color = 'gray';
const archiveName = `${imageFileName}.zip`;
const archivePath = `cypress/fixtures/${archiveName}`;
const imagesFolder = `cypress/fixtures/${imageFileName}`;
const directoryToArchive = imagesFolder;

const jobOptions = {
jobType: 'Ground truth',
Expand Down Expand Up @@ -175,42 +164,77 @@ context('Ground truth jobs', () => {
cy.get('.cvat-conflict-label.cvat-conflict-darken').should('have.length', darkenConflicts);
}

function waitForReport(authKey, rqID) {
cy.request({
function waitForReport(cvat, rqID) {
return new Promise((resolve) => {
function request() {
cvat.server.request(`/api/quality/reports?rq_id=${rqID}`, {
method: 'POST',
}).then((response) => {
if (response.status === 201) {
qualityReportID = response.data.id;
resolve(qualityReportID);
} else {
setTimeout(request, 500);
}
});
}

setTimeout(request, 500);
});
}

function createTaskQualityReport(taskId) {
cy.window().then((window) => window.cvat.server.request('/api/quality/reports', {
method: 'POST',
url: `/api/quality/reports?rq_id=${rqID}`,
headers: {
Authorization: `Token ${authKey}`,
},
body: {
task_id: taskID,
data: {
task_id: taskId,
},
}).then((response) => {
if (response.status === 201) {
qualityReportID = response.body.id;
return;
}
waitForReport(authKey, rqID);
const rqID = response.data.rq_id;
return waitForReport(window.cvat, rqID);
})).then(() => {
cy.visit('/tasks');
cy.get('.cvat-spinner').should('not.exist');
cy.intercept('GET', '/api/quality/reports**').as('getReport');

cy.openTask(taskName);
openQualityTab();
cy.wait('@getReport');
});
}

before(() => {
cy.visit('auth/login');
cy.login();
cy.visit('/tasks');
cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount);
cy.createZipArchive(directoryToArchive, archivePath);
cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName);
cy.openTask(taskName);
cy.url().then((url) => {
taskID = Number(url.split('/').slice(-1)[0].split('?')[0]);
});
cy.get('.cvat-job-item').first().invoke('attr', 'data-row-id').then((val) => {
jobID = val;
});
});

describe(`Testing case "${caseId}"`, () => {
describe('Testing ground truth basics', () => {
const imagesCount = 10;
const imageFileName = 'ground_truth_1';
const width = 800;
const height = 800;
const posX = 10;
const posY = 10;
const color = 'gray';
const archiveName = `${imageFileName}.zip`;
const archivePath = `cypress/fixtures/${archiveName}`;
const imagesFolder = `cypress/fixtures/${imageFileName}`;
const directoryToArchive = imagesFolder;

before(() => {
cy.visit('/tasks');
cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount);
cy.createZipArchive(directoryToArchive, archivePath);
cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName);
cy.openTask(taskName);
cy.url().then((url) => {
taskID = Number(url.split('/').slice(-1)[0].split('?')[0]);
});
cy.get('.cvat-job-item').first().invoke('attr', 'data-row-id').then((val) => {
jobID = val;
});
});

it('Create ground truth job from task page', () => {
cy.createJob({
...jobOptions,
Expand Down Expand Up @@ -303,31 +327,7 @@ context('Ground truth jobs', () => {
});
cy.saveJob();

cy.logout();
cy.getAuthKey().then((res) => {
const authKey = res.body.key;
cy.request({
method: 'POST',
url: '/api/quality/reports',
headers: {
Authorization: `Token ${authKey}`,
},
body: {
task_id: taskID,
},
}).then((response) => {
const rqID = response.body.rq_id;
waitForReport(authKey, rqID);
});
});
cy.login();
cy.visit('/tasks');
cy.get('.cvat-spinner').should('not.exist');
cy.intercept('GET', '/api/quality/reports**').as('getReport');

cy.openTask(taskName);
openQualityTab();
cy.wait('@getReport');
createTaskQualityReport(taskID);
checkCardValue('.cvat-task-mean-annotation-quality', '33.3%');
checkCardValue('.cvat-task-gt-conflicts', '5');
checkCardValue('.cvat-task-issues', '0');
Expand Down Expand Up @@ -397,4 +397,103 @@ context('Ground truth jobs', () => {
cy.checkFrameNum(groundTruthFrames[3]);
});
});

describe('Testing case ground truth job list', () => {
const imagesCount = 20;
const imageFileName = 'ground_truth_2';
const width = 100;
const height = 100;
const posX = 10;
const posY = 10;
const color = 'gray';
const archiveName = `${imageFileName}.zip`;
const archivePath = `cypress/fixtures/${archiveName}`;
const imagesFolder = `cypress/fixtures/${imageFileName}`;
const directoryToArchive = imagesFolder;
let labels = [];

before(() => {
cy.visit('/tasks');
cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount);
cy.createZipArchive(directoryToArchive, archivePath);
cy.createAnnotationTask(
taskName, labelName, attrName,
textDefaultValue, archiveName, false,
{ multiJobs: true, segmentSize: 1 },
);
cy.openTask(taskName);
cy.url().then((url) => {
taskID = Number(url.split('/').slice(-1)[0].split('?')[0]);
});
cy.get('.cvat-job-item').first().invoke('attr', 'data-row-id').then((val) => {
jobID = val;
}).then(() => {
cy.intercept(`/api/labels?**job_id=${jobID}**`).as('getJobLabels');
cy.visit(`/tasks/${taskID}/jobs/${jobID}`);
cy.wait('@getJobLabels').then((interception) => {
labels = interception.response.body.results;
});
});
});

it('Create ground truth job, compute quality report, check jobs table', () => {
cy.window().then((window) => window.cvat.server.request('/api/jobs', {
method: 'POST',
data: {
task_id: taskID,
frame_count: 20,
type: 'ground_truth',
frame_selection_method: 'random_uniform',
},
}).then((response) => {
jobID = response.data.id;
return window.cvat.server.request(`/api/jobs/${jobID}/annotations`, {
method: 'PUT',
data: {
shapes: [],
tracks: [{
label_id: labels[0].id,
frame: 0,
group: 0,
source: 'manual',
attributes: [],
elements: [],
shapes: [{
type: 'rectangle',
occluded: false,
z_order: 0,
rotation: 0,
outside: false,
attributes: [],
frame: 0,
points: [250, 350, 350, 450],
}],
}],
tags: [],
},
});
}).then(() => (
window.cvat.server.request(`/api/jobs/${jobID}`, {
method: 'PATCH',
data: {
stage: 'acceptance',
state: 'completed',
},
})
))).then(() => {
createTaskQualityReport(taskID);
cy.get('.cvat-task-jobs-table .ant-pagination-item').last().invoke('text').then((page) => {
const lastPage = parseInt(page, 10);

for (let i = 0; i < lastPage; i++) {
cy.get('.cvat-task-jobs-table-row').each((row) => {
cy.get(row).should('not.include.text', 'N/A');
});

cy.get('.cvat-task-jobs-table .ant-pagination-next').click();
}
});
});
});
});
});

0 comments on commit 2c5bb69

Please sign in to comment.