Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[#12048] Fix seed, migration and verification script for course verification #13086

Merged
merged 23 commits into from
Apr 24, 2024
Merged

[#12048] Fix seed, migration and verification script for course verification #13086

Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
3e108c7
Add migration and verification script for feedback session
Apr 6, 2024
2275560
Optimize data migration
Apr 6, 2024
dba2272
Fix lint
Apr 6, 2024
64e02d8
Uncomment seed feedback session function
Apr 6, 2024
280eaed
Add counts verification script
Apr 6, 2024
8d8e054
Merge branch 'v9-course-migration' of https://github.com/TEAMMATES/te…
Apr 7, 2024
fd92583
Merge branch 'v9-course-migration' of https://github.com/TEAMMATES/te…
Apr 17, 2024
04f49c6
Merge branch 'v9-course-migration' of https://github.com/TEAMMATES/te…
Apr 21, 2024
27502b9
Merge branch 'v9-course-migration' of https://github.com/TEAMMATES/te…
Apr 22, 2024
72098b3
Merge branch 'v9-course-migration' of https://github.com/TEAMMATES/te…
Apr 22, 2024
89293f4
Fix some small issues
Apr 22, 2024
46f669c
Abstract seed account method
NicolasCwy Apr 23, 2024
cc493e8
Combine non-course and course seed methods
NicolasCwy Apr 23, 2024
0b38691
Fix migration script
NicolasCwy Apr 23, 2024
80bcac0
Add additional logs
NicolasCwy Apr 23, 2024
6c5bf37
Try resolving vulnerabilities
FergusMok Apr 24, 2024
64285e9
Update DataMigrationForCourseEntitySql.java
NicolasCwy Apr 24, 2024
ba8e976
Comment out usage statistics, account requests, notifications, add nu…
FergusMok Apr 24, 2024
a6ba317
Merge with course migration branch
FergusMok Apr 24, 2024
0bfdf42
Merge with course migration branch
FergusMok Apr 24, 2024
a1ecfd6
Merge with course migration branch
FergusMok Apr 24, 2024
bc7baa4
Merge with course migration branch
FergusMok Apr 24, 2024
82a6862
Add stack trace, error messages, and setting of google id
FergusMok Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 77 additions & 44 deletions src/client/java/teammates/client/scripts/sql/DataMigrationForCourseEntitySql.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import teammates.common.util.JsonUtils;
import teammates.storage.sqlentity.Account;
import teammates.storage.sqlentity.BaseEntity;
import teammates.storage.sqlentity.Instructor;
import teammates.storage.sqlentity.Section;
import teammates.storage.sqlentity.Student;
import teammates.storage.sqlentity.Team;
Expand All @@ -48,7 +49,6 @@
import teammates.storage.entity.FeedbackResponse;
import teammates.storage.entity.FeedbackResponseComment;
import teammates.storage.entity.FeedbackSession;
import teammates.storage.entity.Instructor;
import teammates.storage.sqlentity.User;
import teammates.storage.sqlentity.questions.FeedbackConstantSumQuestion.FeedbackConstantSumQuestionDetailsConverter;
import teammates.storage.sqlentity.questions.FeedbackContributionQuestion.FeedbackContributionQuestionDetailsConverter;
Expand Down Expand Up @@ -91,12 +91,18 @@ public class DataMigrationForCourseEntitySql extends DatastoreClient {
AtomicLong numberOfScannedKey;
AtomicLong numberOfUpdatedEntities;

private static final int MAX_RESPONSE_COUNT = -1;

private VerifyCourseEntityAttributes verifier;

public DataMigrationForCourseEntitySql() {
numberOfAffectedEntities = new AtomicLong();
numberOfScannedKey = new AtomicLong();
numberOfUpdatedEntities = new AtomicLong();

entitiesSavingBuffer = new ArrayList<>();

verifier = new VerifyCourseEntityAttributes();

String connectionUrl = ClientProperties.SCRIPT_API_URL;
String username = ClientProperties.SCRIPT_API_NAME;
Expand All @@ -105,8 +111,6 @@ public DataMigrationForCourseEntitySql() {
HibernateUtil.buildSessionFactory(connectionUrl, username, password);
}

private static final int MAX_RESPONSE_COUNT = -1;

public static void main(String[] args) {
new DataMigrationForCourseEntitySql().doOperationRemotely();
}
Expand All @@ -128,26 +132,35 @@ protected void migrateCourse(Course oldCourse) throws Exception {

migrateCourseEntity(newCourse);
flushEntitiesSavingBuffer();
// verifyCourseEntity(newCourse);
// markOldCourseAsMigrated(courseId)

if (!verifier.equals(newCourse, oldCourse)) {
logError("Verification failed for course with id: " + oldCourse.getUniqueId());
return;
}

// TODO: markOldCourseAsMigrated(courseId)

log("Finish migrating course with id: " + oldCourse.getUniqueId());
}

private void migrateCourseEntity(teammates.storage.sqlentity.Course newCourse) {
Map<String, User> userEmailToUserMap = new HashMap<>();
Map<String, Section> sectionNameToSectionMap = migrateSectionChain(newCourse, userEmailToUserMap);
Map<String, User> userGoogleIdToUserMap = new HashMap<>();
Map<String, Instructor> emailToInstructorMap = new HashMap<>();
Map<String, Student> emailToStudentMap = new HashMap<>();
Map<String, Section> sectionNameToSectionMap = migrateSectionChain(newCourse, userGoogleIdToUserMap, emailToStudentMap);
Map<String, teammates.storage.sqlentity.FeedbackSession> feedbackSessionNameToFeedbackSessionMap =
migrateFeedbackChain(newCourse, sectionNameToSectionMap);
migrateInstructorEntities(newCourse, userEmailToUserMap);
migrateUserAccounts(newCourse, userEmailToUserMap);
migrateDeadlineExtensionEntities(newCourse, feedbackSessionNameToFeedbackSessionMap, userEmailToUserMap);
migrateInstructorEntities(newCourse, userGoogleIdToUserMap, emailToInstructorMap);
migrateUserAccounts(newCourse, userGoogleIdToUserMap);
migrateDeadlineExtensionEntities(newCourse, feedbackSessionNameToFeedbackSessionMap, emailToInstructorMap, emailToStudentMap);
}

// methods for migrate section chain ----------------------------------------------------------------------------------
// entities: Section, Team, Student

private Map<String, teammates.storage.sqlentity.Section> migrateSectionChain(
teammates.storage.sqlentity.Course newCourse, Map<String, User> userEmailToUserMap) {
teammates.storage.sqlentity.Course newCourse, Map<String, User> userGoogleIdToUserMap, Map<String, Student> emailToStudentMap) {
log("Migrating section chain");
List<CourseStudent> oldStudents = ofy().load().type(CourseStudent.class).filter("courseId", newCourse.getId())
.list();
Map<String, teammates.storage.sqlentity.Section> sections = new HashMap<>();
Expand All @@ -160,30 +173,33 @@ private Map<String, teammates.storage.sqlentity.Section> migrateSectionChain(
teammates.storage.sqlentity.Section newSection = createSection(newCourse, sectionName);
sections.put(sectionName, newSection);
saveEntityDeferred(newSection);
migrateTeams(newCourse, newSection, stuList, userEmailToUserMap);
migrateTeams(newCourse, newSection, stuList, userGoogleIdToUserMap, emailToStudentMap);
}
return sections;
}

private void migrateTeams(teammates.storage.sqlentity.Course newCourse,
teammates.storage.sqlentity.Section newSection, List<CourseStudent> studentsInSection,
Map<String, User> userEmailToUserMap) {
Map<String, User> userGoogleIdToUserMap, Map<String, Student> emailToStudentMap) {
Map<String, List<CourseStudent>> teamNameToStuMap = studentsInSection.stream()
.collect(Collectors.groupingBy(CourseStudent::getTeamName));
for (Map.Entry<String, List<CourseStudent>> entry : teamNameToStuMap.entrySet()) {
String teamName = entry.getKey();
List<CourseStudent> stuList = entry.getValue();
teammates.storage.sqlentity.Team newTeam = createTeam(newSection, teamName);
saveEntityDeferred(newTeam);
migrateStudents(newCourse, newTeam, stuList, userEmailToUserMap);
migrateStudents(newCourse, newTeam, stuList, userGoogleIdToUserMap, emailToStudentMap);
}
}

private void migrateStudents(teammates.storage.sqlentity.Course newCourse, teammates.storage.sqlentity.Team newTeam,
List<CourseStudent> studentsInTeam, Map<String, User> userEmailToUserMap) {
List<CourseStudent> studentsInTeam, Map<String, User> userGoogleIdToUserMap, Map<String, Student> emailToStudentMap) {
for (CourseStudent oldStudent : studentsInTeam) {
teammates.storage.sqlentity.Student newStudent = createStudent(newCourse, newTeam, oldStudent);
userEmailToUserMap.put(oldStudent.getEmail(), newStudent);
teammates.storage.sqlentity.Student newStudent = migrateStudent(newCourse, newTeam, oldStudent);
emailToStudentMap.put(newStudent.getEmail(), newStudent);
if (oldStudent.getGoogleId() != null) {
userGoogleIdToUserMap.put(oldStudent.getGoogleId(), newStudent);
}
}
}

Expand Down Expand Up @@ -215,7 +231,7 @@ private teammates.storage.sqlentity.Team createTeam(teammates.storage.sqlentity.
return newTeam;
}

private Student createStudent(teammates.storage.sqlentity.Course newCourse,
private Student migrateStudent(teammates.storage.sqlentity.Course newCourse,
teammates.storage.sqlentity.Team newTeam,
CourseStudent oldStudent) {
String truncatedStudentName = truncateToLength255(oldStudent.getName());
Expand All @@ -226,6 +242,7 @@ private Student createStudent(teammates.storage.sqlentity.Course newCourse,
newStudent.setUpdatedAt(oldStudent.getUpdatedAt());
newStudent.setRegKey(oldStudent.getRegistrationKey());
newStudent.setCreatedAt(oldStudent.getCreatedAt());
saveEntityDeferred(newStudent);

return newStudent;
}
Expand All @@ -236,7 +253,7 @@ private Student createStudent(teammates.storage.sqlentity.Course newCourse,
private Map<String, teammates.storage.sqlentity.FeedbackSession> migrateFeedbackChain(
teammates.storage.sqlentity.Course newCourse,
Map<String, Section> sectionNameToSectionMap) {

log("Migrating feedback chain");
Map<String, teammates.storage.sqlentity.FeedbackSession> feedbackSessionNameToFeedbackSessionMap =
new HashMap<>();

Expand Down Expand Up @@ -502,17 +519,20 @@ private teammates.storage.sqlentity.FeedbackResponseComment createFeedbackRespon
// entities: Instructor, DeadlineExtension

private void migrateInstructorEntities(teammates.storage.sqlentity.Course newCourse,
Map<String, User> userEmailToUserMap) {
List<Instructor> oldInstructors = ofy().load().type(Instructor.class).filter("courseId", newCourse.getId())
Map<String, User> userGoogleIdToUserMap, Map<String, Instructor> emailToInstructorMap) {
List<teammates.storage.entity.Instructor> oldInstructors = ofy().load().type(teammates.storage.entity.Instructor.class).filter("courseId", newCourse.getId())
.list();
for (Instructor oldInstructor : oldInstructors) {
teammates.storage.sqlentity.Instructor newInstructor = migrateInstructor(newCourse, oldInstructor);
userEmailToUserMap.put(oldInstructor.getEmail(), newInstructor);
for (teammates.storage.entity.Instructor oldInstructor : oldInstructors) {
Instructor newInstructor = migrateInstructor(newCourse, oldInstructor);
emailToInstructorMap.put(newInstructor.getEmail(), newInstructor);
if (oldInstructor.getGoogleId() != null) {
userGoogleIdToUserMap.put(oldInstructor.getGoogleId(), newInstructor);
}
}
}

private teammates.storage.sqlentity.Instructor migrateInstructor(teammates.storage.sqlentity.Course newCourse,
Instructor oldInstructor) {
private Instructor migrateInstructor(teammates.storage.sqlentity.Course newCourse,
teammates.storage.entity.Instructor oldInstructor) {
InstructorPrivileges newPrivileges;
if (oldInstructor.getInstructorPrivilegesAsText() == null) {
newPrivileges = new InstructorPrivileges(oldInstructor.getRole());
Expand All @@ -537,26 +557,38 @@ private teammates.storage.sqlentity.Instructor migrateInstructor(teammates.stora
newInstructor.setCreatedAt(oldInstructor.getCreatedAt());
newInstructor.setUpdatedAt(oldInstructor.getUpdatedAt());
newInstructor.setRegKey(oldInstructor.getRegistrationKey());
saveEntityDeferred(newInstructor);

return newInstructor;
}

private void migrateDeadlineExtensionEntities(teammates.storage.sqlentity.Course newCourse,
Map<String, teammates.storage.sqlentity.FeedbackSession> feedbackSessionNameToFeedbackSessionMap,
Map<String, User> userEmailToUserMap) {
Map<String, Instructor> emailToInstructorMap, Map<String, Student> emailToStudentMap) {
log("Migrating deadline extension");
List<DeadlineExtension> oldDeadlineExtensions = ofy().load().type(DeadlineExtension.class)
.filter("courseId", newCourse.getId())
.list();

for (DeadlineExtension oldDeadlineExtension : oldDeadlineExtensions) {
User newUser = userEmailToUserMap.get(oldDeadlineExtension.getUserEmail());
if (newUser == null) {
log("User not found for deadline extension: " + oldDeadlineExtension.getUserEmail());
continue;
String userEmail = oldDeadlineExtension.getUserEmail();
String feedbackSessionName = oldDeadlineExtension.getFeedbackSessionName();
teammates.storage.sqlentity.FeedbackSession feedbackSession = feedbackSessionNameToFeedbackSessionMap.get(feedbackSessionName);
User user;
if (oldDeadlineExtension.getIsInstructor()) {
user = emailToInstructorMap.get(userEmail);
if (user == null) {
logError("Instructor not found for deadline extension: " + oldDeadlineExtension);
continue;
}
} else {
user = emailToStudentMap.get(userEmail);
if (user == null) {
logError("Student not found for deadline extension: " + oldDeadlineExtension);
continue;
}
}
migrateDeadlineExtension(oldDeadlineExtension,
feedbackSessionNameToFeedbackSessionMap.get(oldDeadlineExtension.getFeedbackSessionName()),
newUser);
migrateDeadlineExtension(oldDeadlineExtension, feedbackSession, user);
}
}

Expand All @@ -577,28 +609,29 @@ private void migrateDeadlineExtension(DeadlineExtension oldDeadlineExtension,
saveEntityDeferred(newDeadlineExtension);
}

private void migrateUserAccounts(teammates.storage.sqlentity.Course newCourse, Map<String, User> userEmailToUserMap) {
List<Account> newAccounts = getAllAccounts(new ArrayList<String>(userEmailToUserMap.keySet()));
if (newAccounts.size() != userEmailToUserMap.size()) {
log("Mismatch in number of accounts: " + newAccounts.size() + " vs " + userEmailToUserMap.size());
// Associate account to users(students and instructors) who have the same matching google id
private void migrateUserAccounts(teammates.storage.sqlentity.Course newCourse, Map<String, User> userGoogleIdToUserMap) {
List<Account> newAccounts = getAllAccounts(new ArrayList<String>(userGoogleIdToUserMap.keySet()));
if (newAccounts.size() != userGoogleIdToUserMap.size()) {
log("Mismatch in number of accounts: " + newAccounts.size() + " vs " + userGoogleIdToUserMap.size());
}
for (Account account: newAccounts) {
User newUser = userEmailToUserMap.get(account.getEmail());
User newUser = userGoogleIdToUserMap.get(account.getGoogleId());
if (newUser == null) {
log("User not found for account: " + account.getEmail());
log("User not found for account: " + account.getGoogleId());
continue;
}
newUser.setAccount(account);
saveEntityDeferred(account);
saveEntityDeferred(newUser);
}
}

private List<Account> getAllAccounts(List<String> userEmailList) {
private List<Account> getAllAccounts(List<String> userGoogleIds) {
HibernateUtil.beginTransaction();
CriteriaBuilder cb = HibernateUtil.getCriteriaBuilder();
CriteriaQuery<Account> cr = cb.createQuery(Account.class);
Root<Account> courseRoot = cr.from(Account.class);
cr.select(courseRoot).where(cb.in(courseRoot.get("email")).value(userEmailList));
Root<Account> accountRoot = cr.from(Account.class);
cr.select(accountRoot).where(cb.in(accountRoot.get("googleId")).value(userGoogleIds));
List<Account> newAccounts = HibernateUtil.createQuery(cr).getResultList();
HibernateUtil.commitTransaction();
return newAccounts;
Expand Down
Loading