modified proc queue system to retry up to three times before moving the request to the PROC_QUEUE_FAILED table and notifying OI admins

added service to clean PROC_QUEUE_FAILED table

LSL2/STPROC/SERVICE_SERVICES.txt

@@ -27,12 +27,13 @@ Function Service_Services(@Service, @Params)
 
 #pragma precomp SRP_PreCompiler
 
-$insert APP_INSERTS
+$Insert APP_INSERTS
-$insert SERVICE_SETUP
+$Insert SERVICE_SETUP
-$insert SERVICES_EQUATES
+$Insert SERVICES_EQUATES
-$insert RLIST_EQUATES
+$Insert RLIST_EQUATES
-$insert SQL_REQUESTS_EQUATES
+$Insert SQL_REQUESTS_EQUATES
 $Insert PROC_QUEUE_EQUATES
+$Insert PROC_QUEUE_FAILED_EQUATES
 
 Equ Comma$ to ','
 
@@ -175,6 +176,7 @@ Service ProcessProcedureQueue()
             ReadNext RequestKeyID else EOF = True$
         Until EOF or Done
             Lock hProcQueue, RequestKeyID then
+            	DeleteRequest = True$
             	Server        = Environment_Services('GetServer')
             	Done          = True$
                 Database_Services('GetKeyIDLock', 'PROC_QUEUE', RequestKeyId)          
@@ -216,7 +218,16 @@ Service ProcessProcedureQueue()
                     LogData<5> = Params
                     ErrCode    = ''
                     If ( Get_Status(ErrCode) or Error_Services('HasError') ) then
+                    	NumAttempts  = RequestRow<PROC_QUEUE.NUM_ATTEMPTS$> + 1
                     	ErrorMessage = Error_Services('GetMessage')
+                    	If NumAttempts LT 3 then
+                    		DeleteRequest                            = False$
+                    		RequestRow<PROC_QUEUE.NUM_ATTEMPTS$>     = NumAttempts
+							RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
+							RequestRow<PROC_QUEUE.ERROR$>            = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
+							Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyId, RequestRow, True$, False$, False$)
+						end else
+							// Notify OI_SYSADMIN group
 							Recipients                           = ''
 							SentFrom                             = 'SYSTEM'
 							Subject                              = 'Background Procedure Queue Error'
@@ -232,6 +243,13 @@ Service ProcessProcedureQueue()
 							SendToGroup                          = 'OI_SYSADMIN'
 							Parms                                = Recipients:@RM:SentFrom:@RM:Subject:@RM:Message:@RM:AttachWindow:@RM:AttachKey:@RM:SendToGroup
 							obj_Notes('Create',Parms)
+							
+							// Move request to PROC_QUEUE_FAILED
+                    		RequestRow<PROC_QUEUE.NUM_ATTEMPTS$>     = NumAttempts
+							RequestRow<PROC_QUEUE.LAST_ATTEMPT_DTM$> = Datetime()
+							RequestRow<PROC_QUEUE.ERROR$>            = 'Error_Services error message: ':ErrorMessage:'Get_Status error code: ':ErrCode							
+							Database_Services('WriteDataRow', 'PROC_QUEUE_FAILED', RequestKeyId, RequestRow, True$, False$, False$)
+						end
 						LogData<6> = 'Failed'
 						LogData<7> = 'Error_Services error message: ':ErrorMessage:' Get_Status error code: ':ErrCode
 					end else
@@ -239,7 +257,9 @@ Service ProcessProcedureQueue()
 					end
 					Logging_Services('AppendLog', objLog, LogData, @RM, @FM)
                 end
-                Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$) ; // This call should release the lock
+                If DeleteRequest then 
+					Database_Services('DeleteDataRow', 'PROC_QUEUE', RequestKeyId, True$, False$)
+				end
                 Unlock hProcQueue, RequestKeyID else Null
             end
         Repeat
@@ -253,7 +273,6 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
     If Critical EQ '' then Critical = True$
     PostToQueue = True$
     If (ProcedureName NE '') then
-    	If (Params NE '') then
 		If Not(Critical) then
 			// Check if there is already a matching request in the queue before posting another one.
 			Query = 'PROC_NAME':@VM:ProcedureName:@FM
@@ -280,9 +299,6 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
 			RequestRow<PROC_QUEUE.ENTRY_DTM$> = Datetime()
 			Database_Services('WriteDataRow', 'PROC_QUEUE', RequestKeyID, RequestRow, True$, False$, False$)
 		end
-    	end else
-    		Error_Services('Add', 'Error in ':Service:' service. Null Params passed in.')
-    	end
     end else
         Error_Services('Add', 'Error in ':Service:' service. Null ProcedureName passed in.')
     end
@@ -290,6 +306,97 @@ Service PostProcedure(ProcedureName, Params, Critical=BOOLEAN)
 return
 
 
+Service CleanFailedProcQueue(DaysToRetain)
+	
+	hSysLists = Database_Services('GetTableHandle', 'SYSLISTS')
+	Lock hSysLists, ServiceKeyID then	
+		LogPath                = Environment_Services('GetApplicationRootPath') : '\LogFiles\ProcQueue'
+		LogDate                = Oconv(Date(), 'D4/')
+		LogTime                = Oconv(Time(), 'MTS')
+		LogFileName            = LogDate[7, 4] : '-' : LogDate[1, 2] : '-' : LogDate[4, 2] : ' CleanFailedProcQueue Log.csv'
+		Headers                = 'Logging DTM' : @FM : 'Machine' : @FM : 'RequestKeyId' : @FM : 'Procedure' : @FM : 'Params'
+		Headers               := @FM : 'Result' : @FM : 'Error Message'
+		objFailedProcQueueLog  = Logging_Services('NewLog', LogPath, LogFileName, CRLF$, Comma$, Headers, '', False$, False$)
+		LoggingDTM             = LogDate : ' ' : LogTime   ; // Logging DTM	
+		Server                 = Environment_Services('GetServer')
+		
+		LogData    = ''
+		LogData<1> = LoggingDtm
+		LogData<2> = Server
+		LogData<6> = 'Begin CleanFailedProcQueue'
+		Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
+		
+		ErrorMsg    = ''
+		If (DaysToRetain NE '') then
+			If Num(DaysToRetain) then
+				Open 'PROC_QUEUE_FAILED' to hTable then
+					Cutoff = Datetime() - DaysToRetain
+					Select hTable
+					EOF    = False$
+					Loop
+						Readnext Key else EOF = True$
+					Until EOF
+						Read Rec from hTable, Key then
+							EntryDtm = Rec<PROC_QUEUE_FAILED.ENTRY_DTM$>
+							If (EntryDtm LT Cutoff) then
+								Delete hTable, Key then
+									LogData<1>       = LoggingDtm
+									LogData<2>       = Server
+									LogData<3>       = Key
+									LogData<4>       = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
+									ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
+									Convert @VM to '|' in ProcFailedParams
+									LogData<5>       = ProcFailedParams
+									LogData<6>       = 'Removed PROC_QUEUE_FAILED record.'
+									LogData<7>       = Rec<PROC_QUEUE_FAILED.ERROR$>
+									Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)								
+								end else
+									ErrorMsg = 'Error in ':Service:' service. Error deleting PROC_QUEUE_FAILED record ':Key:'.'
+								end
+							end else
+								LogData<1>       = LoggingDtm
+								LogData<2>       = Server
+								LogData<3>       = Key
+								LogData<4>       = Rec<PROC_QUEUE_FAILED.PROC_NAME$>
+								ProcFailedParams = Rec<PROC_QUEUE_FAILED.PARAMS$>
+								Convert @VM to '|' in ProcFailedParams
+								LogData<5>       = ProcFailedParams
+								LogData<6>       = 'Retaining PROC_QUEUE_FAILED record.'
+								LogData<7>       = Rec<PROC_QUEUE_FAILED.ERROR$>
+								Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)								
+							end
+						end else
+							ErrorMsg = 'Error in ':Service:' service. Error reading PROC_QUEUE_FAILED record ':Key:'.'
+						end
+					Repeat
+				end else
+					ErrorMsg = 'Error in ':Service:' service. Error opening PROC_QUEUE_FAILED table.'
+				end
+			end else
+				ErrorMsg = 'Error in ':Service:' service. DaysToRetain must be a number.'
+			end
+		end else
+			ErrorMsg = 'Error in ':Service:' service. Null DaysToRetain value passed in.'
+		end
+		
+		If ErrorMsg NE '' then
+			LogData<1> = LoggingDtm
+			LogData<2> = Server
+			LogData<6> = ErrorMsg
+			Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)			
+		end
+		
+		LogData<1> = LoggingDtm
+		LogData<2> = Server
+		LogData<6> = 'End CleanFailedProcQueue'
+		Logging_Services('AppendLog', objFailedProcQueueLog, LogData, @RM, @FM)
+		
+		Unlock hSysLists, ServiceKeyID else Null
+	end
+	
+end service
+
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 /// Internal GoSubs
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

LSL2/STPROCINS/PROC_QUEUE_EQUATES.txt

@@ -12,6 +12,8 @@ compile insert PROC_QUEUE_EQUATES
    equ PROC_QUEUE.LAST_ATTEMPT_DTM$ to 3
    equ PROC_QUEUE.ERROR$            to 4
    equ PROC_QUEUE.ENTRY_DTM$        to 5
+   equ PROC_QUEUE.NUM_ATTEMPTS$		to 6
 
 #endif
 
+

LSL2/STPROCINS/PROC_QUEUE_FAILED_EQUATES.txt

@@ -0,0 +1,17 @@
+compile insert PROC_QUEUE_FAILED_EQUATES
+/*----------------------------------------
+ Author : Table Create Insert Routine
+ Written : 06/06/2025
+ Description : Insert for Table PROC_QUEUE_FAILED
+----------------------------------------*/
+#ifndef __PROC_QUEUE_FAILED_EQUATES__ 
+#define __PROC_QUEUE_FAILED_EQUATES__ 
+
+   equ PROC_QUEUE_FAILED.PROC_NAME$    to 1
+   equ PROC_QUEUE_FAILED.PARAMS$       to 2
+   equ PROC_QUEUE_FAILED.COMP_DTM$     to 3
+   equ PROC_QUEUE_FAILED.ERROR$        to 4
+   equ PROC_QUEUE_FAILED.ENTRY_DTM$    to 5
+   equ PROC_QUEUE_FAILED.NUM_ATTEMPTS$ to 6
+
+#endif