modified proc queue system to retry up to three times before moving the request to the PROC_QUEUE_FAILED table and notifying OI admins
added service to clean PROC_QUEUE_FAILED table
This commit is contained in:
Infineon\StieberD
Stieber Daniel (CSC FI SPS MESLEO)
parent
2a28c4b743
commit
c4fbf7c1d9
@ -27,12 +27,13 @@ Function Service_Services(@Service, @Params)
|
||||
|
||||
#pragma precomp SRP_PreCompiler
|
||||
|
||||
$insert APP_INSERTS
|
||||
$insert SERVICE_SETUP
|
||||
$insert SERVICES_EQUATES
|
||||
$insert RLIST_EQUATES
|
||||
$insert SQL_REQUESTS_EQUATES
|
||||
$Insert APP_INSERTS
|
||||
$Insert SERVICE_SETUP
|
||||
$Insert SERVICES_EQUATES
|
||||
$Insert RLIST_EQUATES
|
||||
$Insert SQL_REQUESTS_EQUATES
|
||||
$Insert PROC_QUEUE_EQUATES
|
||||
$Insert PROC_QUEUE_FAILED_EQUATES
|
||||
|
||||
Equ Comma$ to ','
|
||||
|
||||
@ -175,10 +176,11 @@ Service ProcessProcedureQueue()
|
||||
ReadNext RequestKeyID else EOF = True$
|
||||
Until EOF or Done
|
||||
Lock hProcQueue, RequestKeyID then
|
||||
Server = Environment_Services('GetServer')
|
||||
Done = True$
|
||||
DeleteRequest = True$
|
||||
Server = Environment_Services('GetServer')
|
||||
Done = True$
|
||||
Database_Services('GetKeyIDLock', 'PROC_QUEUE', RequestKeyId)
|
||||
RequestRow = Database_Services('ReadDataRow', 'PROC_QUEUE', RequestKeyID)
|
||||
RequestRow = Database_Services('ReadDataRow', 'PROC_QUEUE', RequestKeyID)
|
||||
If RequestRow NE '' then
|
||||
Procedure = RequestRow<PROC_QUEUE.PROC_NAME$>
|
||||
Procedure = UCase(Procedure)
|
||||
@ -216,30 +218,48 @@ Service ProcessProcedureQueue()
|
||||
LogData<5> = Params
|
||||
ErrCode = ''
|
||||
If ( Get_Status(ErrCode) or Error_Services('HasError') ) then
|
||||
ErrorMessage = Error_Services('GetMessage')
|
||||
Recipients = ''
|
||||
SentFrom = 'SYSTEM'
|
||||
Subject = 'Background Procedure Queue Error'
|
||||
Message = OConv(Datetime(), 'DT2/^H')
|
||||
Message<2> = 'Error on server ':Server
|
||||
Message<3> = 'Stored Procedure: ':Procedure
|
||||
StatCodes = 'Parameters: ':Params
|
||||
Message<4> = 'Error_Services error message: ':ErrorMessage
|
||||
Message<5> = 'Get_Status error code: ':ErrCode
|
||||
Swap @FM with \0D0A\ in Message
|
||||
AttachWindow = ''
|
||||
AttachKey = ''
|
||||
SendToGroup = 'OI_SYSADMIN'
|
||||
Parms = Recipients:@RM:SentFrom:@RM:Subject:@RM:Message:@RM:AttachWindow:@RM:AttachKey:@RM:SendToGroup
|
||||
obj_Notes('Create',Parms)
|
||||
LogData<6> = 'Failed'
|
||||
LogData<7> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
|
||||
NumAttempts = RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> + 1
|
||||
ErrorMessage = Error_Services('GetMessage')
|
||||
If NumAttempts LT 3 then
|
||||
DeleteRequest = False$
|
||||
RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> = NumAttempts
|
||||
RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
|
||||
RequestRow<PROC_QUEUE.ERROR$> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
|
||||
Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyId, RequestRow, True$, False$, False$)
|
||||
end else
|
||||
// Notify OI_SYSADMIN group
|
||||
Recipients = ''
|
||||
SentFrom = 'SYSTEM'
|
||||
Subject = 'Background Procedure Queue Error'
|
||||
Message = OConv(Datetime(), 'DT2/^H')
|
||||
Message<2> = 'Error on server ':Server
|
||||
Message<3> = 'Stored Procedure: ':Procedure
|
||||
StatCodes = 'Parameters: ':Params
|
||||
Message<4> = 'Error_Services error message: ':ErrorMessage
|
||||
Message<5> = 'Get_Status error code: ':ErrCode
|
||||
Swap @FM with \0D0A\ in Message
|
||||
AttachWindow = ''
|
||||
AttachKey = ''
|
||||
SendToGroup = 'OI_SYSADMIN'
|
||||
Parms = Recipients:@RM:SentFrom:@RM:Subject:@RM:Message:@RM:AttachWindow:@RM:AttachKey:@RM:SendToGroup
|
||||
obj_Notes('Create',Parms)
|
||||
|
||||
// Move request to PROC_QUEUE_FAILED
|
||||
RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> = NumAttempts
|
||||
RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
|
||||
RequestRow<PROC_QUEUE.ERROR$> = 'Error_Services error message: ':ErrorMessage:'Get_Status error code: ':ErrCode
|
||||
Database_Services('WriteDataRow', 'PROC_QUEUE_FAILED', RequestKeyId, RequestRow, True$, False$, False$)
|
||||
end
|
||||
LogData<6> = 'Failed'
|
||||
LogData<7> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
|
||||
end else
|
||||
LogData<6> = 'Success'
|
||||
end
|
||||
Logging_Services('AppendLog', objLog, LogData, @RM, @FM)
|
||||
end
|
||||
Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$) ; // This call should release the lock
|
||||
If DeleteRequest then
|
||||
Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$)
|
||||
end
|
||||
Unlock hProcQueue, RequestKeyID else Null
|
||||
end
|
||||
Repeat
|
||||
@ -253,36 +273,32 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
|
||||
If Critical EQ '' then Critical = True$
|
||||
PostToQueue = True$
|
||||
If (ProcedureName NE '') then
|
||||
If (Params NE '') then
|
||||
If Not(Critical) then
|
||||
// Check if there is already a matching request in the queue before posting another one.
|
||||
Query = 'PROC_NAME':@VM:ProcedureName:@FM
|
||||
Query := 'PARAMS':@VM:Params:@FM
|
||||
Open 'DICT.PROC_QUEUE' to hDict then
|
||||
Keys = ''
|
||||
Option = 'E'
|
||||
Flag = ''
|
||||
Btree.Extract(Query, 'PROC_QUEUE', hDict, Keys, Option, Flag)
|
||||
If Flag EQ 0 then
|
||||
If Keys NE '' then PostToQueue = False$
|
||||
end else
|
||||
Error_Services('Add', 'Error in ':Service:' service. Error calling Btree.Extract.')
|
||||
end
|
||||
If Not(Critical) then
|
||||
// Check if there is already a matching request in the queue before posting another one.
|
||||
Query = 'PROC_NAME':@VM:ProcedureName:@FM
|
||||
Query := 'PARAMS':@VM:Params:@FM
|
||||
Open 'DICT.PROC_QUEUE' to hDict then
|
||||
Keys = ''
|
||||
Option = 'E'
|
||||
Flag = ''
|
||||
Btree.Extract(Query, 'PROC_QUEUE', hDict, Keys, Option, Flag)
|
||||
If Flag EQ 0 then
|
||||
If Keys NE '' then PostToQueue = False$
|
||||
end else
|
||||
Error_Services('Add', 'Error in ':Service:' service. Error opening DICT.PROC_QUEUE.')
|
||||
Error_Services('Add', 'Error in ':Service:' service. Error calling Btree.Extract.')
|
||||
end
|
||||
end else
|
||||
Error_Services('Add', 'Error in ':Service:' service. Error opening DICT.PROC_QUEUE.')
|
||||
end
|
||||
If PostToQueue then
|
||||
RequestKeyID = RTI_CreateGUID()
|
||||
RequestRow = ''
|
||||
RequestRow<PROC_QUEUE.PROC_NAME$> = ProcedureName
|
||||
RequestRow<PROC_QUEUE.PARAMS$> = Params
|
||||
RequestRow<PROC_QUEUE.ENTRY_DTM$> = Datetime()
|
||||
Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyID, RequestRow, True$, False$, False$)
|
||||
end
|
||||
end else
|
||||
Error_Services('Add', 'Error in ':Service:' service. Null Params passed in.')
|
||||
end
|
||||
end
|
||||
If PostToQueue then
|
||||
RequestKeyID = RTI_CreateGUID()
|
||||
RequestRow = ''
|
||||
RequestRow<PROC_QUEUE.PROC_NAME$> = ProcedureName
|
||||
RequestRow<PROC_QUEUE.PARAMS$> = Params
|
||||
RequestRow<PROC_QUEUE.ENTRY_DTM$> = Datetime()
|
||||
Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyID, RequestRow, True$, False$, False$)
|
||||
end
|
||||
end else
|
||||
Error_Services('Add', 'Error in ':Service:' service. Null ProcedureName passed in.')
|
||||
end
|
||||
@ -290,6 +306,97 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
|
||||
return
|
||||
|
||||
|
||||
Service CleanFailedProcQueue(DaysToRetain)
|
||||
|
||||
hSysLists = Database_Services('GetTableHandle', 'SYSLISTS')
|
||||
Lock hSysLists, ServiceKeyID then
|
||||
LogPath = Environment_Services('GetApplicationRootPath') : '\LogFiles\ProcQueue'
|
||||
LogDate = Oconv(Date(), 'D4/')
|
||||
LogTime = Oconv(Time(), 'MTS')
|
||||
LogFileName = LogDate[7, 4] : '-' : LogDate[1, 2] : '-' : LogDate[4, 2] : ' CleanFailedProcQueue Log.csv'
|
||||
Headers = 'Logging DTM' : @FM : 'Machine' : @FM : 'RequestKeyId' : @FM : 'Procedure' : @FM : 'Params'
|
||||
Headers := @FM : 'Result' : @FM : 'Error Message'
|
||||
objFailedProcQueueLog = Logging_Services('NewLog', LogPath, LogFileName, CRLF$, Comma$, Headers, '', False$, False$)
|
||||
LoggingDTM = LogDate : ' ' : LogTime ; // Logging DTM
|
||||
Server = Environment_Services('GetServer')
|
||||
|
||||
LogData = ''
|
||||
LogData<1> = LoggingDtm
|
||||
LogData<2> = Server
|
||||
LogData<6> = 'Begin CleanFailedProcQueue'
|
||||
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
|
||||
|
||||
ErrorMsg = ''
|
||||
If (DaysToRetain NE '') then
|
||||
If Num(DaysToRetain) then
|
||||
Open 'PROC_QUEUE_FAILED' to hTable then
|
||||
Cutoff = Datetime() - DaysToRetain
|
||||
Select hTable
|
||||
EOF = False$
|
||||
Loop
|
||||
Readnext Key else EOF = True$
|
||||
Until EOF
|
||||
Read Rec from hTable, Key then
|
||||
EntryDtm = Rec<PROC_QUEUE_FAILED.ENTRY_DTM$>
|
||||
If (EntryDtm LT Cutoff) then
|
||||
Delete hTable, Key then
|
||||
LogData<1> = LoggingDtm
|
||||
LogData<2> = Server
|
||||
LogData<3> = Key
|
||||
LogData<4> = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
|
||||
ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
|
||||
Convert @VM to '|' in ProcFailedParams
|
||||
LogData<5> = ProcFailedParams
|
||||
LogData<6> = 'Removed PROC_QUEUE_FAILED record.'
|
||||
LogData<7> = Rec<PROC_QUEUE_FAILED.ERROR$>
|
||||
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
|
||||
end else
|
||||
ErrorMsg = 'Error in ':Service:' service. Error deleting PROC_QUEUE_FAILED record ':Key:'.'
|
||||
end
|
||||
end else
|
||||
LogData<1> = LoggingDtm
|
||||
LogData<2> = Server
|
||||
LogData<3> = Key
|
||||
LogData<4> = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
|
||||
ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
|
||||
Convert @VM to '|' in ProcFailedParams
|
||||
LogData<5> = ProcFailedParams
|
||||
LogData<6> = 'Retaining PROC_QUEUE_FAILED record.'
|
||||
LogData<7> = Rec<PROC_QUEUE_FAILED.ERROR$>
|
||||
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
|
||||
end
|
||||
end else
|
||||
ErrorMsg = 'Error in ':Service:' service. Error reading PROC_QUEUE_FAILED record ':Key:'.'
|
||||
end
|
||||
Repeat
|
||||
end else
|
||||
ErrorMsg = 'Error in ':Service:' service. Error opening PROC_QUEUE_FAILED table.'
|
||||
end
|
||||
end else
|
||||
ErrorMsg = 'Error in ':Service:' service. DaysToRetain must be a number.'
|
||||
end
|
||||
end else
|
||||
ErrorMsg = 'Error in ':Service:' service. Null DaysToRetain value passed in.'
|
||||
end
|
||||
|
||||
If ErrorMsg NE '' then
|
||||
LogData<1> = LoggingDtm
|
||||
LogData<2> = Server
|
||||
LogData<6> = ErrorMsg
|
||||
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
|
||||
end
|
||||
|
||||
LogData<1> = LoggingDtm
|
||||
LogData<2> = Server
|
||||
LogData<6> = 'End CleanFailedProcQueue'
|
||||
Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
|
||||
|
||||
Unlock hSysLists, ServiceKeyID else Null
|
||||
end
|
||||
|
||||
end service
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Internal GoSubs
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@ -12,6 +12,8 @@ compile insert PROC_QUEUE_EQUATES
|
||||
equ PROC_QUEUE.LAST_ATTEMPT_DTM$ to 3
|
||||
equ PROC_QUEUE.ERROR$ to 4
|
||||
equ PROC_QUEUE.ENTRY_DTM$ to 5
|
||||
equ PROC_QUEUE.NUM_ATTEMPTS$ to 6
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
17
LSL2/STPROCINS/PROC_QUEUE_FAILED_EQUATES.txt
Normal file
17
LSL2/STPROCINS/PROC_QUEUE_FAILED_EQUATES.txt
Normal file
@ -0,0 +1,17 @@
|
||||
compile insert PROC_QUEUE_FAILED_EQUATES
|
||||
/*----------------------------------------
|
||||
Author : Table Create Insert Routine
|
||||
Written : 06/06/2025
|
||||
Description : Insert for Table PROC_QUEUE_FAILED
|
||||
----------------------------------------*/
|
||||
#ifndef __PROC_QUEUE_FAILED_EQUATES__
|
||||
#define __PROC_QUEUE_FAILED_EQUATES__
|
||||
|
||||
equ PROC_QUEUE_FAILED.PROC_NAME$ to 1
|
||||
equ PROC_QUEUE_FAILED.PARAMS$ to 2
|
||||
equ PROC_QUEUE_FAILED.COMP_DTM$ to 3
|
||||
equ PROC_QUEUE_FAILED.ERROR$ to 4
|
||||
equ PROC_QUEUE_FAILED.ENTRY_DTM$ to 5
|
||||
equ PROC_QUEUE_FAILED.NUM_ATTEMPTS$ to 6
|
||||
|
||||
#endif
|
||||
Loading…
x
Reference in New Issue
Block a user