- prop Fid @ Db.Post.PostContent.BasePostContent, all its derived classes & all its usages

- param `fid` @ `Tieba.Crawl.Saver.Related.ReplyContentImageSaver.Save()`, also affected its usage at `Worker.ProcessImagesInAllReplyContentsWorker.DoWork()`
* replace where clause on the removed prop `(Sub)?ReplyContents.Fid` with correlated subquery from (sub) replies table @ `Worker.PushAllPostContentsIntoSonicWorker.DoWork()`
+ prop `SubReplies`, partial revert 40bf7da @ `Db.CrawlerDbContext`
@ crawler

- prop `Fid` @ `Db.ReplyContentImage`
@ shared
@ c#

Author: n0099

c#/crawler/src/Db/CrawlerDbContext.cs

@@ -21,6 +21,7 @@ public class CrawlerDbContext(ILogger<CrawlerDbContext> logger)
     public DbSet<ReplyPost> Replies => Set<ReplyPost>();
     public DbSet<ReplySignature> ReplySignatures => Set<ReplySignature>();
     public DbSet<ReplyContent> ReplyContents => Set<ReplyContent>();
+    public DbSet<SubReplyPost> SubReplies => Set<SubReplyPost>();
     public DbSet<SubReplyContent> SubReplyContents => Set<SubReplyContent>();
     public DbSet<Forum> Forums => Set<Forum>();
 

c#/crawler/src/Db/Post/PostContent/BasePostContent.cs

@@ -3,6 +3,5 @@ namespace tbm.Crawler.Db.Post.PostContent;
 
 public abstract class BasePostContent : RowVersionedEntity
 {
-    public uint Fid { get; set; }
     public byte[]? ProtoBufBytes { get; set; }
 }

c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs

@@ -18,8 +18,8 @@ public partial class ReplySaver(
             posts => posts.Where(r => Posts.Keys.Contains(r.Pid)));
 
         db.ReplyContents.AddRange(changeSet.NewlyAdded // https://github.com/dotnet/efcore/issues/33945
-            .Select(r => new ReplyContent {Fid = Fid, Pid = r.Pid, ProtoBufBytes = r.Content}));
-        PostSaveHandlers += replyContentImageSaver.Save(db, Fid, changeSet.NewlyAdded);
+            .Select(r => new ReplyContent {Pid = r.Pid, ProtoBufBytes = r.Content}));
+        PostSaveHandlers += replyContentImageSaver.Save(db, changeSet.NewlyAdded);
         PostSaveHandlers += replySignatureSaver.Save(db, changeSet.AllParsed);
 
         return changeSet;

c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs

@@ -16,7 +16,7 @@ public partial class SubReplySaver(
             posts => posts.Where(sr => Posts.Keys.Contains(sr.Spid)));
 
         db.SubReplyContents.AddRange(changeSet.NewlyAdded.Select(sr => // https://github.com/dotnet/efcore/issues/33945
-            new SubReplyContent {Fid = Fid, Spid = sr.Spid, ProtoBufBytes = sr.Content}));
+            new SubReplyContent {Spid = sr.Spid, ProtoBufBytes = sr.Content}));
 
         return changeSet;
     }

c#/crawler/src/Tieba/Crawl/Saver/Related/ReplyContentImageSaver.cs

@@ -27,7 +27,7 @@ public void Dispose()
         }
     }
 
-    public Action Save(CrawlerDbContext db, Fid fid, IEnumerable<ReplyPost.Parsed> replies)
+    public Action Save(CrawlerDbContext db, IEnumerable<ReplyPost.Parsed> replies)
     {
         var replyContentImages = (
                 from r in replies
@@ -37,7 +37,6 @@ from c in r.ContentsProtoBuf
                     ReplyParser.ValidateContentImageFilenameRegex().IsMatch(c.OriginSrc)
                 select new ReplyContentImage
                 {
-                    Fid = fid,
                     Pid = r.Pid,
                     ImageInReply = new()
                     {
@@ -105,7 +104,6 @@ on existingOrNew.UrlFilename equals replyContentImage.ImageInReply.UrlFilename
                 select (existingOrNew, replyContentImage))
             .ForEach(t => t.replyContentImage.ImageInReply = t.existingOrNew);
         var existingReplyContentImages = db.ReplyContentImages.AsNoTracking()
-            .Where(e => e.Fid == fid)
             .FilterByItems(replyContentImages, (existing, newOrExisting) =>
                 existing.Pid == newOrExisting.Pid
                 && existing.ImageInReply.UrlFilename == newOrExisting.ImageInReply.UrlFilename)

c#/crawler/src/Worker/ProcessImagesInAllReplyContentsWorker.cs

@@ -28,7 +28,6 @@ await Transform(
                 readingEntity => readingEntity.Pid,
                 readingEntity => new()
                 {
-                    Fid = fid,
                     Pid = readingEntity.Pid,
                     ProtoBufBytes = readingEntity.ProtoBufBytes,
                     Version = readingEntity.Version
@@ -54,10 +53,9 @@ await Transform(
                         var p = ee.Property(e => e.ProtoBufBytes);
                         p.IsModified = !ByteArrayEqualityComparer.Instance.Equals(p.OriginalValue, p.CurrentValue);
                     });
-                    _ = replyContentImageSaver.Save(writingDb, fid,
+                    _ = replyContentImageSaver.Save(writingDb,
                         replyContentsKeyByPid.Select(pair => new ReplyPost.Parsed
                         {
-                            Fid = fid,
                             Pid = pair.Key,
                             Content = null!,
                             ContentsProtoBuf = pair.Value

c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs

@@ -36,14 +36,16 @@ protected override async Task DoWork(CancellationToken stoppingToken)
             var forumIndex = (index + 1) * 2; // counting from one, including both reply and sub reply
 
             _ = await pusher.Ingest.FlushBucketAsync($"{pusher.CollectionPrefix}replies_content", $"f{fid}");
-            var replyContents = from r in db.ReplyContents.AsNoTracking() where r.Fid == fid select r;
+            var replyContents = from r in db.ReplyContents.AsNoTracking()
+                where db.Replies.Where(e => e.Fid == fid).Select(e => e.Pid).Contains(r.Pid) select r;
             pushedPostCount += PushPostContentsWithTiming(fid, forumIndex - 1, forumCount, "replies",
                 replyCount, totalPostCount, pushedPostCount, replyContents,
                 r => pusher.PushPost(fid, "replies", r.Pid, Helper.ParseThenUnwrapPostContent(r.ProtoBufBytes)), stoppingToken);
             await TriggerConsolidate();
 
             _ = await pusher.Ingest.FlushBucketAsync($"{pusher.CollectionPrefix}subReplies_content", $"f{fid}");
-            var subReplyContents = from sr in db.SubReplyContents.AsNoTracking() where sr.Fid == fid select sr;
+            var subReplyContents = from sr in db.SubReplyContents.AsNoTracking()
+                where db.SubReplies.Where(e => e.Fid == fid).Select(e => e.Spid).Contains(sr.Spid) select sr;
             pushedPostCount += PushPostContentsWithTiming(fid, forumIndex, forumCount, "sub replies",
                 subReplyCount, totalPostCount, pushedPostCount, subReplyContents,
                 sr => pusher.PushPost(fid, "subReplies", sr.Spid, Helper.ParseThenUnwrapPostContent(sr.ProtoBufBytes)), stoppingToken);

c#/imagePipeline/src/ImageBatchConsumingWorker.cs

@@ -218,7 +218,6 @@ IEnumerable<ImageKeyWithMatrix> GetImagesInCurrentFid()
                 // try to know which fid owns current image batch
                 return imageKeysWithMatrix.IntersectBy(
                     from replyContentImage in db.ReplyContentImages.AsNoTracking()
-                    where replyContentImage.Fid == fid
                     where imageKeysWithMatrix
                         .Select(imageKeyWithMatrix => imageKeyWithMatrix.ImageId)
                         .Contains(replyContentImage.ImageId)

c#/shared/src/Db/ReplyContentImage.cs

@@ -5,7 +5,6 @@ namespace tbm.Shared.Db;
 
 public class ReplyContentImage : EntityWithImageId
 {
-    public uint Fid { get; set; }
     [Column(TypeName = "bigint")]
     public ulong Pid { get; set; }
     public required ImageInReply ImageInReply { get; set; }