Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Dev - System Notification in the Event of Data Interface Errors [jp-0002] #181

Merged
merged 1 commit into from
May 31, 2023
Merged

Dev - System Notification in the Event of Data Interface Errors [jp-0002] #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 85 additions & 60 deletions app/Console/Commands/ExportDatabaseToBI.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class ExportDatabaseToBI extends Command

];

protected $task;
protected $success;
protected $failure;
protected $message;
Expand Down Expand Up @@ -117,69 +118,93 @@ private function sendTableDataToDataWarehouse($table_name, $delta_field, $hidden
]);

$this->task = $task;

try {

// Get the latest success job
$last_job = ScheduleJobAudit::where('job_name', $job_name)
->where('status','Completed')
->orderBy('end_time', 'desc')->first();
// Get the latest success job
$last_job = ScheduleJobAudit::where('job_name', $job_name)
->where('status','Completed')
->orderBy('end_time', 'desc')->first();


$last_start_time = $last_job ? $last_job->start_time : '2000-01-01' ;
$last_start_time = $last_job ? $last_job->start_time : '2000-01-01' ;

$this->LogMessage("Table '{$table_name}' Detail to BI (Datawarehouse) start on " . now() );
$this->Logmessage("");
$this->Logmessage("Table Name : '{$table_name}' ");
$this->LogMessage("The Last send completed time : " . $last_start_time);
$this->LogMessage("The schedule Job id : " . $task->id);
$this->LogMessage("The command name : " . $job_name);

// Main Process for each table
$sql = DB::table($table_name)
->when( $last_job && $delta_field, function($q) use($last_start_time, $delta_field, $hidden_fields ) {
return $q->where($delta_field, '>=', $last_start_time);
})
->orderBy('id');
$this->LogMessage("Table '{$table_name}' Detail to BI (Datawarehouse) start on " . now() );
$this->Logmessage("");
$this->Logmessage("Table Name : '{$table_name}' ");
$this->LogMessage("The Last send completed time : " . $last_start_time);
$this->LogMessage("The schedule Job id : " . $task->id);
$this->LogMessage("The command name : " . $job_name);

// Chucking
$row_count = 0;
$sql->chunk(5000, function($chuck) use($task, $table_name, $hidden_fields, $last_job, &$row_count, &$n) {
$this->LogMessage( "Sending table '{$table_name}' batch (5000) - " . ++$n );

//$chuck->makeHidden(['password', 'remember_token']);
if ($hidden_fields) {
foreach($chuck as $item) {
foreach($hidden_fields as $hidden_field) {
// unset($item->password);
unset($item->$hidden_field);
}
// Main Process for each table
$sql = DB::table($table_name)
->when( $last_job && $delta_field, function($q) use($last_start_time, $delta_field, $hidden_fields ) {
return $q->where($delta_field, '>=', $last_start_time);
})
->orderBy('id');

// Chucking
$row_count = 0;
$sql->chunk(5000, function($chuck) use($task, $table_name, $hidden_fields, $last_job, &$row_count, &$n) {
$this->LogMessage( "Sending table '{$table_name}' batch (5000) - " . ++$n );

//$chuck->makeHidden(['password', 'remember_token']);
if ($hidden_fields) {
foreach($chuck as $item) {
foreach($hidden_fields as $hidden_field) {
// unset($item->password);
unset($item->$hidden_field);
}

}
}
}

$pushdata = new stdClass();
$pushdata->table_name = $table_name;
$pushdata->table_data = json_encode($chuck);
$pushdata->delta_ind = $last_job ? "1" : "0";

$result = $this->sendData( $pushdata );
if ($result) {
// Log to the table
foreach($chuck as $row) {
ExportAuditLog::create([
'schedule_job_name' => $task->job_name,
'schedule_job_id' => $task->id,
'to_application' => 'BI',
'table_name' => $table_name,
'row_id' => $row->id,
'row_values' => json_encode($row),
]);

$row_count += 1;
$pushdata = new stdClass();
$pushdata->table_name = $table_name;
$pushdata->table_data = json_encode($chuck);
$pushdata->delta_ind = $last_job ? "1" : "0";

$result = $this->sendData( $pushdata );
if ($result) {
// Log to the table
foreach($chuck as $row) {
ExportAuditLog::create([
'schedule_job_name' => $task->job_name,
'schedule_job_id' => $task->id,
'to_application' => 'BI',
'table_name' => $table_name,
'row_id' => $row->id,
'row_values' => json_encode($row),
]);

$row_count += 1;
}
}

unset($pushdata);
});

} catch (\Exception $ex) {

// log message in system
if ($this->task) {
$this->task->status = 'Error';
$this->task->end_time = Carbon::now();
$this->task->message = $this->task->message . PHP_EOL . $ex->getMessage() . PHP_EOL;
$this->task->save();
}

unset($pushdata);
});

// send out email notification
$notify = new \App\MicrosoftGraph\SendEmailNotification();
$notify->job_id = $this->task ? $this->task->id : null;
$notify->job_name = $job_name;
$notify->error_message = $ex->getMessage();
$notify->send();

// write message to the log
throw new Exception($ex);

}

$this->LogMessage("" );
$this->LogMessage("Success (No of Batch ) - " . $this->success);
Expand All @@ -202,7 +227,7 @@ private function sendTableDataToDataWarehouse($table_name, $delta_field, $hidden

protected function sendData($pushdata) {

try {
// try {

$response = Http::withBasicAuth(
env('ODS_USERNAME'),
Expand All @@ -224,14 +249,14 @@ protected function sendData($pushdata) {
return false;
}

} catch (\Exception $ex) {
// } catch (\Exception $ex) {

// log message in system
$this->status = 'Error';
$this->LogMessage( $ex->getMessage() );
// // log message in system
// $this->status = 'Error';
// $this->LogMessage( $ex->getMessage() );

throw new Exception($ex);
}
// throw new Exception($ex);
// }

}

Expand Down
127 changes: 69 additions & 58 deletions app/Console/Commands/ExportPledgesToPSFT.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace App\Console\Commands;

use Carbon\Carbon;
use Exception;
use App\Models\User;
use App\Models\Pledge;
use App\Models\PayCalendar;
Expand Down Expand Up @@ -36,6 +37,7 @@ class ExportPledgesToPSFT extends Command
protected $description = 'Sending pledge transactions to PeopleSoft via ODS';

/* Source Type is HCM */
protected $task;
protected $message;
protected $status;
protected $bypass_rule_for_testing_purpose;
Expand Down Expand Up @@ -75,40 +77,63 @@ public function handle()
$this->bypass_rule_for_testing_purpose = true;
}

$this->task = ScheduleJobAudit::Create([
'job_name' => $this->signature,
'start_time' => Carbon::now(),
'status' => 'Processing',
]);
try {

$this->task = ScheduleJobAudit::Create([
'job_name' => $this->signature,
'start_time' => Carbon::now(),
'status' => 'Processing',
]);

$this->LogMessage("Process run on " . today()->format('Y-m-d'));
if (!(App::environment('prod'))) {
$this->LogMessage("Bypass Rule (when to send to PSFT) for testing purpose is " . ($this->bypass_rule_for_testing_purpose ? 'Yes' : 'No'));
}
$this->LogMessage( "" );

// Step 1 : Send Campiagn Type data to PeopleSoft access endpoint
$this->LogMessage( now() );
$this->LogMessage("1) Sending Annual Campaign Type pledge data to PeopleSoft");
$this->sendCampaignDonationToPeopleSoft();

// Step 2 : Send Donate Now Pledge data to PeopleSoft access endpoint
$this->LogMessage( "" );
$this->LogMessage("2) Sending Donate Now Type pledge data to PeopleSoft");
$this->sendDonateNowToPeopleSoft();

// Step 3 : Send Special Campaign Pledge data to PeopleSoft access endpoint
$this->LogMessage( "" );
$this->LogMessage("3) Sending Special Campaign Type pledge data to PeopleSoft");
$this->sendSpecialCampaignToPeopleSoft();

// Update the Task Audit log
$this->task->end_time = Carbon::now();
$this->task->status = $this->status;
$this->task->message = $this->message;
$this->task->save();
$this->LogMessage("Process run on " . today()->format('Y-m-d'));
if (!(App::environment('prod'))) {
$this->LogMessage("Bypass Rule (when to send to PSFT) for testing purpose is " . ($this->bypass_rule_for_testing_purpose ? 'Yes' : 'No'));
}
$this->LogMessage( "" );

// Step 1 : Send Campiagn Type data to PeopleSoft access endpoint
$this->LogMessage( now() );
$this->LogMessage("1) Sending Annual Campaign Type pledge data to PeopleSoft");
$this->sendCampaignDonationToPeopleSoft();

// Step 2 : Send Donate Now Pledge data to PeopleSoft access endpoint
$this->LogMessage( "" );
$this->LogMessage("2) Sending Donate Now Type pledge data to PeopleSoft");
$this->sendDonateNowToPeopleSoft();

// Step 3 : Send Special Campaign Pledge data to PeopleSoft access endpoint
$this->LogMessage( "" );
$this->LogMessage("3) Sending Special Campaign Type pledge data to PeopleSoft");
$this->sendSpecialCampaignToPeopleSoft();

// Update the Task Audit log
$this->task->end_time = Carbon::now();
$this->task->status = $this->status;
$this->task->message = $this->message;
$this->task->save();

} catch (\Exception $ex) {

// log message in system
if ($this->task) {
$this->task->status = 'Error';
$this->task->end_time = Carbon::now();
$this->task->message = $ex->getMessage() . PHP_EOL;
$this->task->save();
}

// send out email notification
$notify = new \App\MicrosoftGraph\SendEmailNotification();
$notify->job_id = $this->task ? $this->task->id : null;
$notify->job_name = $this->signature;
$notify->error_message = $ex->getMessage();
$notify->send();

// write message to the log
throw new Exception($ex);

}


}
Expand Down Expand Up @@ -517,41 +542,27 @@ private function sendSpecialCampaignToPeopleSoft() {

protected function sendData($pushdata) {

try {

$response = Http::withBasicAuth(
env('ODS_USERNAME'),
env('ODS_TOKEN')
)->withBody( json_encode($pushdata), 'application/json')
->post( env('ODS_OUTBOUND_PLEDGE_PSFT_ENDPOINT') );

if ($response->successful()) {
$this->success += 1;
return true;

} else {

// log message in system
$this->status = 'Error';
$this->LogMessage( "(Error) - Data - " . json_encode($pushdata) );
$this->LogMessage( " - " . $response->status() . ' - ' . $response->body() );
$response = Http::withBasicAuth(
env('ODS_USERNAME'),
env('ODS_TOKEN')
)->withBody( json_encode($pushdata), 'application/json')
->post( env('ODS_OUTBOUND_PLEDGE_PSFT_ENDPOINT') );

$this->failure += 1;
if ($response->successful()) {
$this->success += 1;
return true;

return false;
}

} catch (\Exception $ex) {
} else {

// log message in system
$this->status = 'Error';
$this->LogMessage( "(Error) - " . json_encode($pushdata) );
$this->LogMessage( " " - $ex->getMessage() );

throw new Exception($ex);
$this->LogMessage( "(Error) - Data - " . json_encode($pushdata) );
$this->LogMessage( " - " . $response->status() . ' - ' . $response->body() );

}
$this->failure += 1;

return false;
}

}

Expand Down
Loading