phares3757/open-insight
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modified proc queue system to retry up to three times before moving the request to the PROC_QUEUE_FAILED table and notifying OI admins

Browse Source 
added service to clean PROC_QUEUE_FAILED table
This commit is contained in:
Infineon\StieberD 2025-06-06 16:09:48 -07:00 committed by Stieber Daniel (CSC FI SPS MESLEO)
parent 2a28c4b743
commit c4fbf7c1d9
3 changed files with 180 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
LSL2/STPROC/SERVICE_SERVICES.txt
View File

@ -27,12 +27,13 @@ Function Service_Services(@Service, @Params)
#pragma precomp SRP_PreCompiler #pragma precomp SRP_PreCompiler
$insert APP_INSERTS $Insert APP_INSERTS
$insert SERVICE_SETUP $Insert SERVICE_SETUP
$insert SERVICES_EQUATES $Insert SERVICES_EQUATES
$insert RLIST_EQUATES $Insert RLIST_EQUATES
$insert SQL_REQUESTS_EQUATES $Insert SQL_REQUESTS_EQUATES
$Insert PROC_QUEUE_EQUATES $Insert PROC_QUEUE_EQUATES
$Insert PROC_QUEUE_FAILED_EQUATES
Equ Comma$ to ',' Equ Comma$ to ','
@ -175,10 +176,11 @@ Service ProcessProcedureQueue()
ReadNext RequestKeyID else EOF = True$ ReadNext RequestKeyID else EOF = True$
Until EOF or Done Until EOF or Done
Lock hProcQueue, RequestKeyID then Lock hProcQueue, RequestKeyID then
Server = Environment_Services('GetServer') DeleteRequest = True$
Done = True$ Server = Environment_Services('GetServer')
Done = True$
Database_Services('GetKeyIDLock', 'PROC_QUEUE', RequestKeyId) Database_Services('GetKeyIDLock', 'PROC_QUEUE', RequestKeyId)
RequestRow = Database_Services('ReadDataRow', 'PROC_QUEUE', RequestKeyID) RequestRow = Database_Services('ReadDataRow', 'PROC_QUEUE', RequestKeyID)
If RequestRow NE '' then If RequestRow NE '' then
Procedure = RequestRow<PROC_QUEUE.PROC_NAME$> Procedure = RequestRow<PROC_QUEUE.PROC_NAME$>
Procedure = UCase(Procedure) Procedure = UCase(Procedure)
@ -216,30 +218,48 @@ Service ProcessProcedureQueue()
LogData<5> = Params LogData<5> = Params
ErrCode = '' ErrCode = ''
If ( Get_Status(ErrCode) or Error_Services('HasError') ) then If ( Get_Status(ErrCode) or Error_Services('HasError') ) then
ErrorMessage = Error_Services('GetMessage') NumAttempts = RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> + 1
Recipients = '' ErrorMessage = Error_Services('GetMessage')
SentFrom = 'SYSTEM' If NumAttempts LT 3 then
Subject = 'Background Procedure Queue Error' DeleteRequest = False$
Message = OConv(Datetime(), 'DT2/^H') RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> = NumAttempts
Message<2> = 'Error on server ':Server RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
Message<3> = 'Stored Procedure: ':Procedure RequestRow<PROC_QUEUE.ERROR$> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
StatCodes = 'Parameters: ':Params Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyId, RequestRow, True$, False$, False$)
Message<4> = 'Error_Services error message: ':ErrorMessage end else
Message<5> = 'Get_Status error code: ':ErrCode // Notify OI_SYSADMIN group
Swap @FM with \0D0A\ in Message Recipients = ''
AttachWindow = '' SentFrom = 'SYSTEM'
AttachKey = '' Subject = 'Background Procedure Queue Error'
SendToGroup = 'OI_SYSADMIN' Message = OConv(Datetime(), 'DT2/^H')
Parms = Recipients:@RM:SentFrom:@RM:Subject:@RM:Message:@RM:AttachWindow:@RM:AttachKey:@RM:SendToGroup Message<2> = 'Error on server ':Server
obj_Notes('Create',Parms) Message<3> = 'Stored Procedure: ':Procedure
LogData<6> = 'Failed' StatCodes = 'Parameters: ':Params
LogData<7> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode Message<4> = 'Error_Services error message: ':ErrorMessage
Message<5> = 'Get_Status error code: ':ErrCode
Swap @FM with \0D0A\ in Message
AttachWindow = ''
AttachKey = ''
SendToGroup = 'OI_SYSADMIN'
Parms = Recipients:@RM:SentFrom:@RM:Subject:@RM:Message:@RM:AttachWindow:@RM:AttachKey:@RM:SendToGroup
obj_Notes('Create',Parms)
// Move request to PROC_QUEUE_FAILED
RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> = NumAttempts
RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
RequestRow<PROC_QUEUE.ERROR$> = 'Error_Services error message: ':ErrorMessage:'Get_Status error code: ':ErrCode
Database_Services('WriteDataRow', 'PROC_QUEUE_FAILED', RequestKeyId, RequestRow, True$, False$, False$)
end
LogData<6> = 'Failed'
LogData<7> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
end else end else
LogData<6> = 'Success' LogData<6> = 'Success'
end end
Logging_Services('AppendLog', objLog, LogData, @RM, @FM) Logging_Services('AppendLog', objLog, LogData, @RM, @FM)
end end
Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$) ; // This call should release the lock If DeleteRequest then
Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$)
end
Unlock hProcQueue, RequestKeyID else Null Unlock hProcQueue, RequestKeyID else Null
end end
Repeat Repeat
@ -253,36 +273,32 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
If Critical EQ '' then Critical = True$ If Critical EQ '' then Critical = True$
PostToQueue = True$ PostToQueue = True$
If (ProcedureName NE '') then If (ProcedureName NE '') then
If (Params NE '') then If Not(Critical) then
If Not(Critical) then // Check if there is already a matching request in the queue before posting another one.
// Check if there is already a matching request in the queue before posting another one. Query = 'PROC_NAME':@VM:ProcedureName:@FM
Query = 'PROC_NAME':@VM:ProcedureName:@FM Query := 'PARAMS':@VM:Params:@FM
Query := 'PARAMS':@VM:Params:@FM Open 'DICT.PROC_QUEUE' to hDict then
Open 'DICT.PROC_QUEUE' to hDict then Keys = ''
Keys = '' Option = 'E'
Option = 'E' Flag = ''
Flag = '' Btree.Extract(Query, 'PROC_QUEUE', hDict, Keys, Option, Flag)
Btree.Extract(Query, 'PROC_QUEUE', hDict, Keys, Option, Flag) If Flag EQ 0 then
If Flag EQ 0 then If Keys NE '' then PostToQueue = False$
If Keys NE '' then PostToQueue = False$
end else
Error_Services('Add', 'Error in ':Service:' service. Error calling Btree.Extract.')
end
end else end else
Error_Services('Add', 'Error in ':Service:' service. Error opening DICT.PROC_QUEUE.') Error_Services('Add', 'Error in ':Service:' service. Error calling Btree.Extract.')
end end
end else
Error_Services('Add', 'Error in ':Service:' service. Error opening DICT.PROC_QUEUE.')
end end
If PostToQueue then end
RequestKeyID = RTI_CreateGUID() If PostToQueue then
RequestRow = '' RequestKeyID = RTI_CreateGUID()
RequestRow<PROC_QUEUE.PROC_NAME$> = ProcedureName RequestRow = ''
RequestRow<PROC_QUEUE.PARAMS$> = Params RequestRow<PROC_QUEUE.PROC_NAME$> = ProcedureName
RequestRow<PROC_QUEUE.ENTRY_DTM$> = Datetime() RequestRow<PROC_QUEUE.PARAMS$> = Params
Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyID, RequestRow, True$, False$, False$) RequestRow<PROC_QUEUE.ENTRY_DTM$> = Datetime()
end Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyID, RequestRow, True$, False$, False$)
end else end
Error_Services('Add', 'Error in ':Service:' service. Null Params passed in.')
end
end else end else
Error_Services('Add', 'Error in ':Service:' service. Null ProcedureName passed in.') Error_Services('Add', 'Error in ':Service:' service. Null ProcedureName passed in.')
end end
@ -290,6 +306,97 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
return return
Service CleanFailedProcQueue(DaysToRetain)
hSysLists = Database_Services('GetTableHandle', 'SYSLISTS')
Lock hSysLists, ServiceKeyID then
LogPath = Environment_Services('GetApplicationRootPath') : '\LogFiles\ProcQueue'
LogDate = Oconv(Date(), 'D4/')
LogTime = Oconv(Time(), 'MTS')
LogFileName = LogDate[7, 4] : '-' : LogDate[1, 2] : '-' : LogDate[4, 2] : ' CleanFailedProcQueue Log.csv'
Headers = 'Logging DTM' : @FM : 'Machine' : @FM : 'RequestKeyId' : @FM : 'Procedure' : @FM : 'Params'
Headers := @FM : 'Result' : @FM : 'Error Message'
objFailedProcQueueLog = Logging_Services('NewLog', LogPath, LogFileName, CRLF$, Comma$, Headers, '', False$, False$)
LoggingDTM = LogDate : ' ' : LogTime ; // Logging DTM
Server = Environment_Services('GetServer')
LogData = ''
LogData<1> = LoggingDtm
LogData<2> = Server
LogData<6> = 'Begin CleanFailedProcQueue'
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
ErrorMsg = ''
If (DaysToRetain NE '') then
If Num(DaysToRetain) then
Open 'PROC_QUEUE_FAILED' to hTable then
Cutoff = Datetime() - DaysToRetain
Select hTable
EOF = False$
Loop
Readnext Key else EOF = True$
Until EOF
Read Rec from hTable, Key then
EntryDtm = Rec<PROC_QUEUE_FAILED.ENTRY_DTM$>
If (EntryDtm LT Cutoff) then
Delete hTable, Key then
LogData<1> = LoggingDtm
LogData<2> = Server
LogData<3> = Key
LogData<4> = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
Convert @VM to '|' in ProcFailedParams
LogData<5> = ProcFailedParams
LogData<6> = 'Removed PROC_QUEUE_FAILED record.'
LogData<7> = Rec<PROC_QUEUE_FAILED.ERROR$>
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
end else
ErrorMsg = 'Error in ':Service:' service. Error deleting PROC_QUEUE_FAILED record ':Key:'.'
end
end else
LogData<1> = LoggingDtm
LogData<2> = Server
LogData<3> = Key
LogData<4> = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
Convert @VM to '|' in ProcFailedParams
LogData<5> = ProcFailedParams
LogData<6> = 'Retaining PROC_QUEUE_FAILED record.'
LogData<7> = Rec<PROC_QUEUE_FAILED.ERROR$>
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
end
end else
ErrorMsg = 'Error in ':Service:' service. Error reading PROC_QUEUE_FAILED record ':Key:'.'
end
Repeat
end else
ErrorMsg = 'Error in ':Service:' service. Error opening PROC_QUEUE_FAILED table.'
end
end else
ErrorMsg = 'Error in ':Service:' service. DaysToRetain must be a number.'
end
end else
ErrorMsg = 'Error in ':Service:' service. Null DaysToRetain value passed in.'
end
If ErrorMsg NE '' then
LogData<1> = LoggingDtm
LogData<2> = Server
LogData<6> = ErrorMsg
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
end
LogData<1> = LoggingDtm
LogData<2> = Server
LogData<6> = 'End CleanFailedProcQueue'
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
Unlock hSysLists, ServiceKeyID else Null
end
end service
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// Internal GoSubs /// Internal GoSubs
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

2
LSL2/STPROCINS/PROC_QUEUE_EQUATES.txt
View File

@ -12,6 +12,8 @@ compile insert PROC_QUEUE_EQUATES
equ PROC_QUEUE.LAST_ATTEMPT_DTM$ to 3 equ PROC_QUEUE.LAST_ATTEMPT_DTM$ to 3
equ PROC_QUEUE.ERROR$ to 4 equ PROC_QUEUE.ERROR$ to 4
equ PROC_QUEUE.ENTRY_DTM$ to 5 equ PROC_QUEUE.ENTRY_DTM$ to 5
equ PROC_QUEUE.NUM_ATTEMPTS$ to 6
#endif #endif

17
LSL2/STPROCINS/PROC_QUEUE_FAILED_EQUATES.txt Normal file
View File

@ -0,0 +1,17 @@
compile insert PROC_QUEUE_FAILED_EQUATES
/*----------------------------------------
Author : Table Create Insert Routine
Written : 06/06/2025
Description : Insert for Table PROC_QUEUE_FAILED
----------------------------------------*/
#ifndef __PROC_QUEUE_FAILED_EQUATES__
#define __PROC_QUEUE_FAILED_EQUATES__
equ PROC_QUEUE_FAILED.PROC_NAME$ to 1
equ PROC_QUEUE_FAILED.PARAMS$ to 2
equ PROC_QUEUE_FAILED.COMP_DTM$ to 3
equ PROC_QUEUE_FAILED.ERROR$ to 4
equ PROC_QUEUE_FAILED.ENTRY_DTM$ to 5
equ PROC_QUEUE_FAILED.NUM_ATTEMPTS$ to 6
#endif