* now will stop retrying to update _latestReplyPostedAtCheckpoints[fid] and crawl these pages after the configured CrawlerLocks.Thread.MaxRetryTimes times @ Tieba.Crawl.CrawlPost.CrawlThreads()

* now will always decrease `RequesterTcs` when the caught exception is `TiebaException` @ `Tieba.Crawl.Facade.CrawlFacade.LogException()`
* fix violations of Roslyn analyzer rule `SA1103` @ `Worker.PushAllPostContentsIntoSonicWorker.DoWork()`
@ c#/crawler

Author: n0099

c#/crawler/src/Tieba/Crawl/CrawlPost.cs

@@ -7,6 +7,7 @@ namespace tbm.Crawler.Tieba.Crawl;
 using SavedThreadsList = IReadOnlyCollection<SaverChangeSet<ThreadPost, ThreadPost.Parsed>>;
 
 public class CrawlPost(
+    IConfiguration config,
     Func<Owned<CrawlerDbContext.New>> dbContextFactory,
     Func<Owned<ThreadLateCrawlFacade.New>> threadLateCrawlFacadeFactory,
     Func<Owned<ThreadCrawlFacade.New>> threadCrawlFacadeFactory,
@@ -20,10 +21,6 @@ public async Task<SavedThreadsList> CrawlThreads
         (string forumName, Fid fid, CancellationToken stoppingToken = default)
     {
         stoppingToken.ThrowIfCancellationRequested();
-        var savedThreads = new List<SaverChangeSet<ThreadPost, ThreadPost.Parsed>>();
-        Time minLatestReplyPostedAt;
-        Page crawlingPage = 0;
-
         if (!_latestReplyPostedAtCheckpoints.TryGetValue(fid, out var maxLatestReplyPostedAtOccurInPreviousCrawl))
         { // get the largest value of field latestReplyPostedAt in all stored threads of this forum
             // this approach is not as accurate as extracting the last thread in the response list
@@ -34,6 +31,11 @@ public async Task<SavedThreadsList> CrawlThreads
                 .Where(t => t.Fid == fid)
                 .Max(th => (Time?)th.LatestReplyPostedAt) ?? Time.MaxValue;
         }
+        var savedThreads = new List<SaverChangeSet<ThreadPost, ThreadPost.Parsed>>();
+        var minLatestReplyPostedAt = Time.MaxValue;
+        Page crawlingPage = 0;
+        FailureCount failureCount = 0;
+        var maxRetry = config.GetSection("CrawlerLocks:Thread").GetValue("MaxRetryTimes", 5);
         do
         {
             crawlingPage++;
@@ -47,13 +49,19 @@ public async Task<SavedThreadsList> CrawlThreads
                 var threadsLatestReplyPostedAt = currentPageChangeSet.AllParsed
                     .Select(th => th.LatestReplyPostedAt).ToList();
                 minLatestReplyPostedAt = threadsLatestReplyPostedAt.Min();
-                if (crawlingPage == 1)
-                    _latestReplyPostedAtCheckpoints[fid] = threadsLatestReplyPostedAt.Max();
+                var maxLatestReplyPostedAt = threadsLatestReplyPostedAt.Max();
+                if (!_latestReplyPostedAtCheckpoints.TryGetValue(fid, out var checkpoint)
+                    || checkpoint < maxLatestReplyPostedAt)
+                    _latestReplyPostedAtCheckpoints[fid] = maxLatestReplyPostedAt;
             }
-            else
+            else if (failureCount < maxRetry)
             { // retry this page
                 crawlingPage--;
-                minLatestReplyPostedAt = Time.MaxValue;
+                failureCount++;
+            }
+            else if (failureCount >= maxRetry)
+            {
+                minLatestReplyPostedAt = Time.MinValue;
             }
         } while (minLatestReplyPostedAt > maxLatestReplyPostedAtOccurInPreviousCrawl);
 

c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs

@@ -202,20 +202,19 @@ private async Task<bool> LogException(
 
             if (e is TiebaException te)
             {
-                if (!te.ShouldSilent) Logger.LogWarning("TiebaException: {} {}",
-                    string.Join(' ', e.GetInnerExceptions().Select(ex => ex.Message)),
-                    SharedHelper.UnescapedJsonSerialize(e.Data));
+                RequesterTcs.Decrease();
+                if (!te.ShouldSilent)
+                    Logger.LogWarning("TiebaException: {} {}",
+                        string.Join(' ', e.GetInnerExceptions().Select(ex => ex.Message)),
+                        SharedHelper.UnescapedJsonSerialize(e.Data));
             }
             else
             {
                 Logger.LogError(e, "Exception");
             }
 
             if (e is not TiebaException {ShouldRetry: false})
-            {
                 locks.AcquireFailed(lockId, page, (FailureCount)(previousFailureCount + 1));
-                RequesterTcs.Decrease();
-            }
 
             try
             {

c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs

@@ -36,16 +36,20 @@ protected override async Task DoWork(CancellationToken stoppingToken)
             var forumIndex = (index + 1) * 2; // counting from one, including both reply and sub reply
 
             _ = await pusher.Ingest.FlushBucketAsync($"{pusher.CollectionPrefix}replies_content", $"f{fid}");
-            var replyContents = from r in db.ReplyContents.AsNoTracking()
-                where db.Replies.Where(e => e.Fid == fid).Select(e => e.Pid).Contains(r.Pid) select r;
+            var replyContents =
+                from r in db.ReplyContents.AsNoTracking()
+                where db.Replies.Where(e => e.Fid == fid).Select(e => e.Pid).Contains(r.Pid)
+                select r;
             pushedPostCount += PushPostContentsWithTiming(fid, forumIndex - 1, forumCount, "replies",
                 replyCount, totalPostCount, pushedPostCount, replyContents,
                 r => pusher.PushPost(fid, "replies", r.Pid, Helper.ParseThenUnwrapPostContent(r.ProtoBufBytes)), stoppingToken);
             await TriggerConsolidate();
 
             _ = await pusher.Ingest.FlushBucketAsync($"{pusher.CollectionPrefix}subReplies_content", $"f{fid}");
-            var subReplyContents = from sr in db.SubReplyContents.AsNoTracking()
-                where db.SubReplies.Where(e => e.Fid == fid).Select(e => e.Spid).Contains(sr.Spid) select sr;
+            var subReplyContents =
+                from sr in db.SubReplyContents.AsNoTracking()
+                where db.SubReplies.Where(e => e.Fid == fid).Select(e => e.Spid).Contains(sr.Spid)
+                select sr;
             pushedPostCount += PushPostContentsWithTiming(fid, forumIndex, forumCount, "sub replies",
                 subReplyCount, totalPostCount, pushedPostCount, subReplyContents,
                 sr => pusher.PushPost(fid, "subReplies", sr.Spid, Helper.ParseThenUnwrapPostContent(sr.ProtoBufBytes)), stoppingToken);