phares3757/open-insight
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modified copy and delete record to sql routines to lock the queue record before attempting to prevent long running copy or delete calls form being processed more than once and wasting resources.

Browse Source
This commit is contained in:
Infineon\StieberD
2025-07-03 14:03:39 -07:00
parent 5ca3778719
commit 98ac420462
5 changed files with 354 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
LSL2/STPROC/COPY_BACKLOG_RECORDS_TO_SQL.txt
View File

@ -1,6 +1,4 @@
Compile subroutine Copy_Backlog_Records_To_SQL(VOID) Compile subroutine Copy_Backlog_Records_To_SQL(VOID)
#pragma precomp SRP_PreCompiler
/*****************************************************************************\ /*****************************************************************************\
This is a utility function. For each record found in SQL_BACKLOG, it makes This is a utility function. For each record found in SQL_BACKLOG, it makes
sure the identified record is copied to SQL. Then the backlog record is sure the identified record is copied to SQL. Then the backlog record is
@ -10,14 +8,16 @@ Compile subroutine Copy_Backlog_Records_To_SQL(VOID)
------- -------
08/10/2010 KRF Original Programmer 08/10/2010 KRF Original Programmer
\*****************************************************************************/ \*****************************************************************************/
#pragma precomp SRP_PreCompiler
$Insert APP_INSERTS $Insert APP_INSERTS
Declare subroutine Copy_Record_To_SQL, Delete_Record_From_SQL, SRP_TcpClient, Copy_Pending_Records_To_Sql Declare subroutine Copy_Record_To_SQL, Delete_Record_From_SQL, SRP_TcpClient, Copy_Pending_Records_To_Sql
Declare function GetTickCount, Database_Services, SRP_TcpClient, Datetime, Environment_Services Declare function GetTickCount, Database_Services, SRP_TcpClient, Datetime, Environment_Services
hSysLists = Database_Services('GetTableHandle', 'SYSLISTS') Main:
Lock hSysLists, 'Copy_Backlog_Records_To_SQL' then
hSysLists = Database_Services('GetTableHandle', 'SYSLISTS')
Lock hSysLists, 'Copy_Backlog_Records_To_SQL' then
Copy_Pending_Records_To_Sql() Copy_Pending_Records_To_Sql()
@ -86,8 +86,9 @@ Lock hSysLists, 'Copy_Backlog_Records_To_SQL' then
If Skip EQ False$ then If Skip EQ False$ then
TcpClientHandle = 0 TcpClientHandle = 0
ServerIP = Environment_Services('GetApplicationRootIP') ServerIP = Environment_Services('GetApplicationRootIP')
ServerPort = Environment_Services('GetScrapeServerPort')
Convert '\' to '' in ServerIP Convert '\' to '' in ServerIP
If SRP_TcpClient(TcpClientHandle, 'CONNECT', ServerIP, '20779') then If SRP_TcpClient(TcpClientHandle, 'CONNECT', ServerIP, ServerPort) then
SRP_TcpClient(TcpClientHandle, 'SEND_SES', Command) SRP_TcpClient(TcpClientHandle, 'SEND_SES', Command)
SRP_TcpClient(TcpClientHandle, 'CLOSE_SES') SRP_TcpClient(TcpClientHandle, 'CLOSE_SES')
end end
@ -97,8 +98,7 @@ Lock hSysLists, 'Copy_Backlog_Records_To_SQL' then
end end
end end
Unlock hSysLists, 'Copy_Backlog_Records_To_SQL' else Null Unlock hSysLists, 'Copy_Backlog_Records_To_SQL' else Null
end end
Return Return

40
LSL2/STPROC/COPY_PENDING_RECORDS_TO_SQL.txt
View File

@ -1,6 +1,4 @@
Compile subroutine Copy_Pending_Records_To_SQL(VOID) Compile subroutine Copy_Pending_Records_To_SQL(VOID)
#pragma precomp SRP_PreCompiler
/*****************************************************************************\ /*****************************************************************************\
This is a utility function. For each record found in SQL_PENDING, it makes This is a utility function. For each record found in SQL_PENDING, it makes
sure the identified record is copied to SQL. sure the identified record is copied to SQL.
@ -9,28 +7,34 @@ Compile subroutine Copy_Pending_Records_To_SQL(VOID)
------- -------
02/06/2023 DJS Adapted from Copy_Backlog_Records_To_SQL 02/06/2023 DJS Adapted from Copy_Backlog_Records_To_SQL
\*****************************************************************************/ \*****************************************************************************/
#pragma precomp SRP_PreCompiler
$Insert APP_INSERTS $Insert APP_INSERTS
EQU FIVE_MINUTES$ to 0.003472
EQU SECONDS_PER_DAY$ to 86400
EQU THIRTY_MINUTES$ to 1800
Declare subroutine Copy_Record_To_SQL, Delete_Record_From_SQL, SRP_TcpClient, Database_Services Declare subroutine Copy_Record_To_SQL, Delete_Record_From_SQL, SRP_TcpClient, Database_Services
Declare function GetTickCount, Database_Services, SRP_TcpClient, Datetime, Database_Services, Environment_Services Declare function GetTickCount, Database_Services, SRP_TcpClient, Datetime, Database_Services, Environment_Services
// This only needs to be called every once in a while (e.g. 5 minutes ~ 0.003472) Main:
Run = False$
CurrDtm = Datetime() // This only needs to be called every once in a while (e.g. 5 minutes ~ 0.003472)
LastRunDtm = Database_Services('ReadDataRow', 'APP_INFO', 'SQL_PENDING_LAST_RUNTIME') Run = False$
If LastRunDtm NE '' then CurrDtm = Datetime()
LastRunDtm = Database_Services('ReadDataRow', 'APP_INFO', 'SQL_PENDING_LAST_RUNTIME')
If LastRunDtm NE '' then
TimeDiff = CurrDtm - LastRunDtm TimeDiff = CurrDtm - LastRunDtm
If TimeDiff GT 0.003472 then If TimeDiff GT FIVE_MINUTES$ then
Run = True$ Run = True$
LastRunDtm = CurrDtm LastRunDtm = CurrDtm
end end
end else end else
Run = True$ Run = True$
LastRunDtm = CurrDtm LastRunDtm = CurrDtm
end end
If Run then If Run then
Database_Services('WriteDataRow', 'APP_INFO', 'SQL_PENDING_LAST_RUNTIME', LastRunDtm) Database_Services('WriteDataRow', 'APP_INFO', 'SQL_PENDING_LAST_RUNTIME', LastRunDtm)
hSysLists = Database_Services('GetTableHandle', 'SYSLISTS') hSysLists = Database_Services('GetTableHandle', 'SYSLISTS')
Lock hSysLists, 'Copy_Pending_Records_To_SQL' then Lock hSysLists, 'Copy_Pending_Records_To_SQL' then
@ -48,9 +52,9 @@ If Run then
Read Rec from pTable, Key then Read Rec from pTable, Key then
TransDtm = Rec<1> TransDtm = Rec<1>
ElapSecs = (CurrDtm - TransDtm) * 86400 ElapSecs = (CurrDtm - TransDtm) * SECONDS_PER_DAY$
If ElapSecs LT 1800 then If ElapSecs LT THIRTY_MINUTES$ then
// Retry recent pending transactions (1800 seconds = 30 minutes) // Retry recent pending transactions (i.e., those submitted in the last 30 minutes)
Table = Key[1, "*"] Table = Key[1, "*"]
Locate Table in NonCriticalTables using @FM setting Dummy then Locate Table in NonCriticalTables using @FM setting Dummy then
ID = Key[Col2() + 1, Len(Key)] ID = Key[Col2() + 1, Len(Key)]
@ -81,9 +85,10 @@ If Run then
TcpClientHandle = 0 TcpClientHandle = 0
ServerIP = Environment_Services('GetApplicationRootIP') ServerIP = Environment_Services('GetApplicationRootIP')
ServerPort = Environment_Services('GetScrapeServerPort')
Convert '\' to '' in ServerIP Convert '\' to '' in ServerIP
If SRP_TcpClient(TcpClientHandle, 'CONNECT', ServerIP, '20779') then If SRP_TcpClient(TcpClientHandle, 'CONNECT', ServerIP, ServerPort) then
SRP_TcpClient(TcpClientHandle, 'SEND_SES', Command) SRP_TcpClient(TcpClientHandle, 'SEND_SES', Command)
SRP_TcpClient(TcpClientHandle, 'CLOSE_SES') SRP_TcpClient(TcpClientHandle, 'CLOSE_SES')
end end
@ -99,8 +104,7 @@ If Run then
end end
Unlock hSysLists, 'Copy_Pending_Records_To_SQL' else Null Unlock hSysLists, 'Copy_Pending_Records_To_SQL' else Null
end end
end end
Return Return

41
LSL2/STPROC/COPY_RECORD_TO_SQL.txt
View File

@ -22,20 +22,29 @@ Compile function Copy_Record_To_SQL(Table, Key, LogError, pKey)
09/23/2010 KRF Added logic to set @DICT, @ID, and @RECORD so handlers 09/23/2010 KRF Added logic to set @DICT, @ID, and @RECORD so handlers
can use symbolics can use symbolics
\*****************************************************************************/ \*****************************************************************************/
$insert Microsoft_Ado_Equates $Insert APP_INSERTS
$Insert MICROSOFT_ADO_EQUATES
If Assigned(LogError) else LogError = 0
Declare subroutine SRP_Com, Sql_Services Declare subroutine SRP_Com, Sql_Services
Declare function SRP_Com, Environment_Services, Datetime, Sql_Services, Error_Services, Database_Services Declare function SRP_Com, Environment_Services, Datetime, Sql_Services, Error_Services, Database_Services, Unassigned
Ans = ""
// Make sure table is uppercase Main:
Convert @LOWER_CASE to @UPPER_CASE in Table
TablesToSkip = Database_Services('ReadDataRow', 'APP_INFO', 'SQL_TABLES_TO_SKIP') If Unassigned(LogError) then LogError = False$
Locate Table in TablesToSkip by "AL" using "," setting ListPos else
Ans = ""
Done = False$
// Make sure table is uppercase
Convert @LOWER_CASE to @UPPER_CASE in Table
TablesToSkip = Database_Services('ReadDataRow', 'APP_INFO', 'SQL_TABLES_TO_SKIP')
Locate Table in TablesToSkip by "AL" using "," setting ListPos else
Open 'SQL_PENDING' to pTable then
If pKey NE '' then Lock pTable, pKey else Done = True$
If Not(Done) then
// The expected name of the handler // The expected name of the handler
Handler = "COPY_":Table:"_RECORD_TO_SQL" Handler = "COPY_":Table:"_RECORD_TO_SQL"
@ -56,9 +65,8 @@ Locate Table in TablesToSkip by "AL" using "," setting ListPos else
If Ans EQ '' then If Ans EQ '' then
If Assigned(pKey) then If Assigned(pKey) then
If pKey NE '' then If pKey NE '' then
Open 'SQL_PENDING' to pTable then Delete pTable, pKey else Null
Delete pTable, pKey else null Unlock pTable, pKey else Null
end
end end
end end
end end
@ -68,7 +76,8 @@ Locate Table in TablesToSkip by "AL" using "," setting ListPos else
end end
end end
end else end else
Ans = 'Unable to open connection to SQL server. SQL connection error: ':SRP_Com(Connection, "ERROR") Ans = 'Unable to open connection to SQL server. SQL connection error: '
Ans := SRP_Com(Connection, "ERROR")
end end
SRP_Com(Connection, "CALL", "Close") SRP_Com(Connection, "CALL", "Close")
SRP_Com(Connection, "RELEASE") SRP_Com(Connection, "RELEASE")
@ -111,9 +120,9 @@ Locate Table in TablesToSkip by "AL" using "," setting ListPos else
end end
end end
end end
end end
end
end
Return Ans Return Ans

52
LSL2/STPROC/DELETE_RECORD_FROM_SQL.txt
View File

@ -18,23 +18,30 @@ Compile function Delete_Record_From_SQL(Table, Key, LogError, pKey)
------- -------
09/15/2010 KRF Original Programmer 09/15/2010 KRF Original Programmer
\*****************************************************************************/ \*****************************************************************************/
$Insert APP_INSERTS
$insert Microsoft_Ado_Equates $insert MICROSOFT_ADO_EQUATES
If Assigned(LogError) else LogError = 0
Declare subroutine Sql_Services, SRP_COM Declare subroutine Sql_Services, SRP_COM
Declare function Sql_Services, Error_Services, SRP_COM, Environment_Services Declare function Sql_Services, Error_Services, SRP_COM, Environment_Services, Unassigned
Ans = ""
// Make sure table is uppercase Main:
Convert @LOWER_CASE to @UPPER_CASE in Table
// The expected name of the handler If Unassigned(LogError) else LogError = False$
Handler = "DELETE_":Table:"_RECORD_FROM_SQL" Ans = ""
Done = False$
// Make sure table is uppercase
Convert @LOWER_CASE to @UPPER_CASE in Table
// Find the stored procedure that handles this table Open 'SQL_PENDING' to pTable then
Open "SYSOBJ" to hSysObj then
If pKey NE '' then Lock pTable, pKey else Done = True$
If Not(Done) then
// The expected name of the handler
Handler = "DELETE_":Table:"_RECORD_FROM_SQL"
// Find the stored procedure that handles this table
Open "SYSOBJ" to hSysObj then
Read ObjCode from hSysObj, "$":Handler:"*LSL2" then Read ObjCode from hSysObj, "$":Handler:"*LSL2" then
// Connect to the ODBC/ADO // Connect to the ODBC/ADO
If SRP_Com(Connection, "CREATE", "ADODB.Connection") then If SRP_Com(Connection, "CREATE", "ADODB.Connection") then
@ -46,9 +53,8 @@ Open "SYSOBJ" to hSysObj then
Ans = Function(@Handler(Connection, Key)) Ans = Function(@Handler(Connection, Key))
If Ans EQ '' then If Ans EQ '' then
If Assigned(pKey) then If Assigned(pKey) then
Open 'SQL_PENDING' to pTable then Delete pTable, pKey else Null
Delete pTable, pKey else null Unlock pTable, pKey else Null
end
end end
end end
end end
@ -61,9 +67,9 @@ Open "SYSOBJ" to hSysObj then
Ans = "Unable to create ADO connection." Ans = "Unable to create ADO connection."
end end
end end
end end
If LogError AND Len(Ans) then If LogError AND Len(Ans) then
Open "SQL_ERROR" to hSqlError then Open "SQL_ERROR" to hSqlError then
Read Errors from hSqlError, Date() then Read Errors from hSqlError, Date() then
Errors := @FM:Table:@VM:Key:@VM:Ans Errors := @FM:Table:@VM:Key:@VM:Ans
@ -73,11 +79,12 @@ If LogError AND Len(Ans) then
Write Errors to hSqlError, Date() Write Errors to hSqlError, Date()
end end
end end
end end
// Always log the result // Always log the result
If Assigned(pKey) then If Assigned(pKey) then
If (pKey NE '') then
Open 'SQL_LOG' to hLog then Open 'SQL_LOG' to hLog then
If Ans EQ '' then If Ans EQ '' then
Result = 'PROCESSED' Result = 'PROCESSED'
@ -92,7 +99,10 @@ If Assigned(pKey) then
end end
Write Log to hLog, CurrDate Write Log to hLog, CurrDate
end end
end end
end
end
end
Return Ans Return Ans

13
LSL2/STPROC/ENVIRONMENT_SERVICES.txt
View File

@ -565,8 +565,19 @@ Service GetServiceManagerPort()
end service end service
Service GetScrapeServerPort()
FilePath = Drive():'\SRPEngineServerScrape.ini'
OSRead IniFile from FilePath then
CharIndex = Index(IniFile, 'Port', 1)
Line = IniFile[CharIndex, 'F':CRLF$]
Response = Trim(Line[-1, 'B='])
end
end service
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Internal GoSubs // Internal GoSubs
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////