Skip to content

Commit 90c14be

Browse files
MarioFinaleMarioFinale
committed
Lower API Rate Request rate
1 parent eb53166 commit 90c14be

2 files changed

Lines changed: 184 additions & 122 deletions

File tree

‎MWBot.net/APIHandler.vb‎

Lines changed: 165 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Namespace WikiBot
1818
Private _botPassword As String = String.Empty
1919
Private _apiUri As Uri
2020
Private _userAgent As String = "MWBot.net/" & MwBotVersion & " (http://es.wikipedia.org/wiki/User_talk:MarioFinale) .NET/MONO"
21-
Private _requestDelay As Double = 100
21+
Private _requestDelay As Double = 250
2222
Private _exponentialBackOffDelayMs As Integer = 3000
2323

2424
#Region "Properties"
@@ -247,28 +247,27 @@ Namespace WikiBot
247247
Return GetDataAndResult(pageUri, New CookieContainer)
248248
End Function
249249

250-
''' <summary>Realiza una solicitud de tipo GET a un recurso web y retorna el texto.</summary>
251-
''' <param name="pageUri">URL absoluta del recurso web.</param>
252-
''' <param name="Cookies">Cookies sobre los que se trabaja.</param>
250+
''' <summary>
251+
''' Realiza una solicitud de tipo GET a un recurso web y retorna el texto.
252+
''' Incluye manejo de rate limits (429 y mensaje textual de Wikimedia).
253+
''' </summary>
254+
''' <param name="pageURI">URI absoluta del recurso web.</param>
255+
''' <param name="cookies">Contenedor de cookies para la solicitud.</param>
253256
Public Function GetDataAndResult(ByVal pageUri As Uri, ByRef cookies As CookieContainer) As String
254-
Dim tryCount As Integer = 0
255-
Dim delay As Integer = _exponentialBackOffDelayMs
257+
If pageUri Is Nothing Then Throw New ArgumentNullException(NameOf(pageUri), "Null uri")
256258

257-
Do Until tryCount = MaxRetry
258-
259-
If pageUri Is Nothing Then
260-
Throw New ArgumentNullException(NameOf(pageUri), "Null uri")
261-
End If
259+
Dim tryCount As Integer = 0
260+
Const MaxAttempts As Integer = 6 ' Puedes ajustar según tu MaxRetry existente
262261

263-
If cookies Is Nothing Then
264-
cookies = New CookieContainer
265-
End If
262+
Do While tryCount < MaxAttempts
263+
If cookies Is Nothing Then cookies = New CookieContainer()
266264

267-
Dim RequestDelayInMS As Double = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
268-
While (RequestDelayInMS < _requestDelay) 'Limit post requests per second
265+
' === Throttling (respetar _requestDelay) ===
266+
Dim requestDelayMs As Double = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
267+
While requestDelayMs < _requestDelay
269268
Thread.Sleep(1)
270269
SyncLock RequestLock
271-
RequestDelayInMS = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
270+
requestDelayMs = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
272271
End SyncLock
273272
End While
274273
SyncLock RequestLock
@@ -277,44 +276,76 @@ Namespace WikiBot
277276

278277
Dim tempcookies As CookieContainer = cookies
279278

280-
Dim encoding As New Text.UTF8Encoding
281-
Dim handler As HttpClientHandler = New HttpClientHandler With {
279+
Dim handler As New HttpClientHandler With {
282280
.CookieContainer = cookies,
283281
.UseCookies = True
284282
}
285-
Dim client As HttpClient = New HttpClient(handler)
286-
client.DefaultRequestHeaders.UserAgent.ParseAdd(_userAgent)
287-
client.DefaultRequestHeaders.Connection.ParseAdd("keep-alive")
288-
client.DefaultRequestHeaders.Add("Method", "GET")
289-
Dim response As String = Nothing
290283

291-
Try
292-
Dim message As Task(Of HttpResponseMessage) = client.GetAsync(pageUri)
293-
Dim res As HttpResponseMessage = message.Result
294-
Dim theaders As Headers.HttpResponseHeaders = res.Headers
295-
response = res.Content.ReadAsStringAsync.Result()
296-
tempcookies.Add(cookies.GetCookies(pageUri))
297-
298-
Catch ex As System.Net.WebException
299-
tryCount += 1
300-
delay = delay * 2 ' exponential backoff
301-
Thread.Sleep(delay)
302-
#Disable Warning CA1031
303-
Catch ex2 As Exception
304-
tryCount += 1
305-
delay = delay * 2 ' exponential backoff
306-
Thread.Sleep(delay)
307-
#Enable Warning CA1031
308-
Finally
309-
client.Dispose()
310-
End Try
311-
If Not response Is Nothing Then
312-
cookies = tempcookies
313-
Return AdaptEncoding(response)
314-
End If
315-
Return Nothing
284+
Using client As New HttpClient(handler)
285+
client.DefaultRequestHeaders.UserAgent.ParseAdd(_userAgent)
286+
client.DefaultRequestHeaders.Connection.ParseAdd("keep-alive")
287+
288+
Try
289+
Dim responseTask As Task(Of HttpResponseMessage) = client.GetAsync(pageUri)
290+
Dim res As HttpResponseMessage = responseTask.Result
291+
292+
' === RATE LIMIT DETECTION ===
293+
If res.StatusCode = HttpStatusCode.TooManyRequests OrElse
294+
Not res.IsSuccessStatusCode Then
295+
296+
Dim bodyPreview As String = res.Content.ReadAsStringAsync().Result
297+
298+
If res.StatusCode = HttpStatusCode.TooManyRequests OrElse
299+
bodyPreview.Contains("too many requests to the API", StringComparison.OrdinalIgnoreCase) OrElse
300+
bodyPreview.Contains("rate limit", StringComparison.OrdinalIgnoreCase) OrElse
301+
bodyPreview.Contains("Wikimedia_APIs/Rate_limits", StringComparison.OrdinalIgnoreCase) Then
302+
303+
Dim retryAfterSeconds As Integer = 10 ' valor por defecto seguro
304+
Dim retryValues As IEnumerable(Of String) = Nothing
305+
If res.Headers.TryGetValues("Retry-After", retryValues) Then
306+
Dim headerValue As String = retryValues.FirstOrDefault()
307+
If Not String.IsNullOrEmpty(headerValue) Then
308+
Integer.TryParse(headerValue, retryAfterSeconds)
309+
End If
310+
End If
311+
312+
retryAfterSeconds = Math.Max(retryAfterSeconds, 5) ' mínimo 5 segundos
313+
314+
EventLogger.Log(String.Format("API Rate Limit (429) detectado. Esperando {0} segundos...", retryAfterSeconds), "ApiHandler")
315+
Thread.Sleep(retryAfterSeconds * 1000)
316+
317+
tryCount += 1
318+
Continue Do ' reintentar
319+
End If
320+
321+
' Otro error HTTP
322+
EventLogger.EX_Log(String.Format("HTTP Error {0}: {1}", CInt(res.StatusCode), bodyPreview), "ApiHandler")
323+
Return Nothing
324+
End If
325+
326+
' === ÉXITO ===
327+
Dim response As String = res.Content.ReadAsStringAsync().Result
328+
tempcookies.Add(cookies.GetCookies(pageUri))
329+
cookies = tempcookies
330+
331+
Return AdaptEncoding(response)
332+
333+
Catch ex As WebException
334+
tryCount += 1
335+
EventLogger.EX_Log("WebException en GET: " & ex.Message, "ApiHandler")
336+
Catch ex2 As Exception
337+
tryCount += 1
338+
EventLogger.EX_Log("Excepción en GET: " & ex2.Message, "ApiHandler")
339+
End Try
340+
End Using
341+
342+
' Backoff exponencial entre reintentos
343+
Dim backoff As Integer = _exponentialBackOffDelayMs * (tryCount + 1)
344+
Thread.Sleep(backoff)
316345
Loop
317-
Throw New MaxRetriesExeption
346+
347+
EventLogger.EX_Log("Máximo de reintentos alcanzado en GET.", "ApiHandler")
348+
Throw New MaxRetriesExeption()
318349
End Function
319350

320351
''' <summary>
@@ -436,23 +467,23 @@ Namespace WikiBot
436467
Return PostDataAndGetResult(pageUri, postData, ApiCookies)
437468
End Function
438469

439-
''' <summary>Realiza una solicitud de tipo POST a un recurso web y retorna el texto.</summary>
440-
''' <param name="pageUri">URL absoluta del recurso web.</param>
470+
''' <summary>
471+
''' Realiza una solicitud de tipo POST a un recurso web y retorna el texto.
472+
''' Incluye manejo de rate limits (429 y mensaje textual de Wikimedia).
473+
''' </summary>
474+
''' <param name="pageURI">URI absoluta del recurso web.</param>
441475
''' <param name="postData">Cadena de texto que se envia en el POST.</param>
476+
''' <param name="cookies">Contenedor de cookies para la solicitud.</param>
477+
''' <param name="retrycount">Número de intentos de reintentar la solicitud.</param>
442478
Public Function PostDataAndGetResult(pageUri As Uri, postData As String, ByRef cookies As CookieContainer, Optional retrycount As Integer = 0) As String
479+
If pageUri Is Nothing Then Throw New ArgumentNullException(NameOf(pageUri), "Empty uri.")
480+
If String.IsNullOrEmpty(postData) Then Return String.Empty
443481

444482
Dim delay As Integer = _exponentialBackOffDelayMs
445483

446-
If pageUri Is Nothing Then
447-
Throw New ArgumentNullException(NameOf(pageUri), "Empty uri.")
448-
End If
449-
450-
If cookies Is Nothing Then
451-
cookies = New CookieContainer
452-
End If
453-
484+
' Throttling
454485
Dim RequestDelayInMS As Double = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
455-
While (RequestDelayInMS < _requestDelay) 'Limit post requests per second
486+
While RequestDelayInMS < _requestDelay
456487
Thread.Sleep(1)
457488
SyncLock RequestLock
458489
RequestDelayInMS = (Date.UtcNow - LastRequestTimestamp).TotalMilliseconds
@@ -466,49 +497,85 @@ Namespace WikiBot
466497

467498
Dim encoding As New Text.UTF8Encoding
468499
Dim byteData As Byte() = encoding.GetBytes(postData)
469-
Dim handler As HttpClientHandler = New HttpClientHandler With {
500+
501+
Dim handler As New HttpClientHandler With {
470502
.CookieContainer = cookies,
471503
.UseCookies = True
472504
}
473505

474-
Dim client As HttpClient = New HttpClient(handler)
475-
Dim content As StringContent = New StringContent(postData)
476-
content.Headers.Add("Method", "POST")
477-
content.Headers.ContentType = Headers.MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded")
478-
content.Headers.ContentLength = byteData.Length
479-
client.DefaultRequestHeaders.UserAgent.ParseAdd(_userAgent)
480-
client.DefaultRequestHeaders.Connection.ParseAdd("keep-alive")
481-
client.DefaultRequestHeaders.Add("Method", "POST")
482-
client.Timeout = New TimeSpan(0, 0, 30)
483-
Dim response As String = Nothing
484-
Try
485-
Dim message As Task(Of HttpResponseMessage) = client.PostAsync(pageUri, content)
486-
Dim res As HttpResponseMessage = message.Result
487-
Dim theaders As Headers.HttpResponseHeaders = res.Headers
488-
response = res.Content.ReadAsStringAsync.Result()
489-
tempcookies.Add(cookies.GetCookies(pageUri))
490-
Catch ex As System.Net.WebException
491-
If retrycount < 3 Then
492-
Thread.Sleep(_exponentialBackOffDelayMs * (retrycount + 1)) ' exponential backoff
493-
Return PostDataAndGetResult(pageUri, postData, cookies, retrycount + 1)
494-
End If
495-
#Disable Warning CA1031
496-
Catch ex2 As Exception
497-
If retrycount < 3 Then
498-
Thread.Sleep(_exponentialBackOffDelayMs * (retrycount + 1)) ' exponential backoff
499-
Return PostDataAndGetResult(pageUri, postData, cookies, retrycount + 1)
500-
End If
501-
#Enable Warning CA1031
502-
Finally
503-
client.Dispose()
504-
content.Dispose()
505-
End Try
506-
If Not response Is Nothing Then
507-
cookies = tempcookies
508-
Return AdaptEncoding(response)
509-
End If
506+
Using client As New HttpClient(handler)
507+
Dim content As New StringContent(postData)
508+
content.Headers.ContentType = Headers.MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded")
509+
content.Headers.ContentLength = byteData.Length
510+
511+
client.DefaultRequestHeaders.UserAgent.ParseAdd(_userAgent)
512+
client.DefaultRequestHeaders.Connection.ParseAdd("keep-alive")
513+
client.Timeout = New TimeSpan(0, 0, 30)
514+
515+
Try
516+
Dim message As Task(Of HttpResponseMessage) = client.PostAsync(pageUri, content)
517+
Dim res As HttpResponseMessage = message.Result
518+
519+
' === RATE LIMIT DETECTION ===
520+
If res.StatusCode = HttpStatusCode.TooManyRequests OrElse Not res.IsSuccessStatusCode Then
521+
Dim bodyPreview As String = res.Content.ReadAsStringAsync().Result
522+
523+
If res.StatusCode = HttpStatusCode.TooManyRequests OrElse
524+
bodyPreview.Contains("too many requests to the API", StringComparison.OrdinalIgnoreCase) OrElse
525+
bodyPreview.Contains("rate limit", StringComparison.OrdinalIgnoreCase) OrElse
526+
bodyPreview.Contains("Wikimedia_APIs/Rate_limits", StringComparison.OrdinalIgnoreCase) Then
527+
528+
Dim retryAfterSeconds As Integer = 10
529+
Dim retryValues As IEnumerable(Of String) = Nothing
530+
If res.Headers.TryGetValues("Retry-After", retryValues) Then
531+
Dim headerValue As String = retryValues.FirstOrDefault()
532+
If Not String.IsNullOrEmpty(headerValue) Then
533+
Integer.TryParse(headerValue, retryAfterSeconds)
534+
End If
535+
End If
536+
retryAfterSeconds = Math.Max(retryAfterSeconds, 5)
537+
538+
EventLogger.Log(String.Format("API Rate Limit (429) detectado en POST. Esperando {0} segundos...", retryAfterSeconds), "ApiHandler")
539+
Thread.Sleep(retryAfterSeconds * 1000)
540+
541+
If retrycount < 5 Then
542+
Return PostDataAndGetResult(pageUri, postData, cookies, retrycount + 1)
543+
Else
544+
EventLogger.EX_Log("Máximo de reintentos por rate limit alcanzado.", "ApiHandler")
545+
Return Nothing
546+
End If
547+
End If
548+
549+
EventLogger.EX_Log(String.Format("HTTP Error {0}: {1}", CInt(res.StatusCode), bodyPreview), "ApiHandler")
550+
Return Nothing
551+
End If
552+
553+
' === ÉXITO ===
554+
Dim response As String = res.Content.ReadAsStringAsync().Result
555+
tempcookies.Add(cookies.GetCookies(pageUri))
556+
cookies = tempcookies
557+
558+
Return AdaptEncoding(response)
559+
560+
Catch ex As WebException
561+
If retrycount < 3 Then
562+
Thread.Sleep(_exponentialBackOffDelayMs * (retrycount + 1))
563+
Return PostDataAndGetResult(pageUri, postData, cookies, retrycount + 1)
564+
End If
565+
EventLogger.EX_Log("WebException en POST: " & ex.Message, "ApiHandler")
566+
Catch ex2 As Exception
567+
If retrycount < 3 Then
568+
Thread.Sleep(_exponentialBackOffDelayMs * (retrycount + 1))
569+
Return PostDataAndGetResult(pageUri, postData, cookies, retrycount + 1)
570+
End If
571+
EventLogger.EX_Log("Excepción en POST: " & ex2.Message, "ApiHandler")
572+
Finally
573+
content.Dispose()
574+
End Try
575+
End Using
510576
Return Nothing
511577
End Function
578+
512579
End Class
513580

514581
End Namespace

0 commit comments

Comments
 (0)