|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# |
| 3 | +# This source code is licensed under the MIT license found in the |
| 4 | +# LICENSE file in the root directory of this source tree. |
| 5 | + |
| 6 | +"""Generate DLRM medium and large TorchScript models with random weights. |
| 7 | +
|
| 8 | +For benchmarking, trained weights aren't needed — the compute pattern |
| 9 | +(embedding lookups, MLP forward passes, feature interactions) is identical |
| 10 | +regardless of weight values. These models are architecture-compatible with |
| 11 | +the existing dlrm_small.pt model used by FeedSim's DLRM inference path. |
| 12 | +
|
| 13 | +Usage: |
| 14 | + python3 generate_dlrm_models.py <output_dir> |
| 15 | +""" |
| 16 | + |
| 17 | +import os |
| 18 | +import sys |
| 19 | + |
| 20 | +import torch |
| 21 | +from torch import nn |
| 22 | + |
| 23 | + |
| 24 | +class DLRM(nn.Module): |
| 25 | + """Simplified DLRM model matching production inference patterns.""" |
| 26 | + |
| 27 | + def __init__( |
| 28 | + self, |
| 29 | + emb_dim: int = 64, |
| 30 | + num_dense: int = 13, |
| 31 | + num_sparse: int = 26, |
| 32 | + max_emb_rows: int = 250000, |
| 33 | + bottom_mlp_dims: list = None, |
| 34 | + top_mlp_dims: list = None, |
| 35 | + ): |
| 36 | + super().__init__() |
| 37 | + if bottom_mlp_dims is None: |
| 38 | + bottom_mlp_dims = [256, 128, emb_dim] |
| 39 | + if top_mlp_dims is None: |
| 40 | + top_mlp_dims = [256, 128, 1] |
| 41 | + |
| 42 | + # Bottom MLP: dense features -> embedding dimension |
| 43 | + layers = [] |
| 44 | + in_dim = num_dense |
| 45 | + for out_dim in bottom_mlp_dims: |
| 46 | + layers.append(nn.Linear(in_dim, out_dim)) |
| 47 | + layers.append(nn.ReLU()) |
| 48 | + in_dim = out_dim |
| 49 | + self.bottom_mlp = nn.Sequential(*layers) |
| 50 | + |
| 51 | + # Embedding tables for sparse features |
| 52 | + emb_sizes = [ |
| 53 | + 40000000, |
| 54 | + 39060, |
| 55 | + 17295, |
| 56 | + 7424, |
| 57 | + 20265, |
| 58 | + 3, |
| 59 | + 7122, |
| 60 | + 1543, |
| 61 | + 63, |
| 62 | + 40000000, |
| 63 | + 3067956, |
| 64 | + 405282, |
| 65 | + 10, |
| 66 | + 2209, |
| 67 | + 11938, |
| 68 | + 155, |
| 69 | + 4, |
| 70 | + 976, |
| 71 | + 14, |
| 72 | + 40000000, |
| 73 | + 40000000, |
| 74 | + 40000000, |
| 75 | + 590152, |
| 76 | + 12973, |
| 77 | + 108, |
| 78 | + 36, |
| 79 | + ] |
| 80 | + # Use min(actual_size, max_emb_rows) to control model size |
| 81 | + self.embeddings = nn.ModuleList( |
| 82 | + [ |
| 83 | + nn.EmbeddingBag(min(s, max_emb_rows), emb_dim, mode="sum") |
| 84 | + for s in emb_sizes[:num_sparse] |
| 85 | + ] |
| 86 | + ) |
| 87 | + |
| 88 | + # Top MLP: interaction output -> prediction. ReLU only on hidden |
| 89 | + # layers; the final Linear feeds raw logits into sigmoid in forward(). |
| 90 | + n = 1 + num_sparse # bottom_mlp output + embedding outputs |
| 91 | + interaction_size = emb_dim + (n * (n - 1)) // 2 |
| 92 | + top_layers = [] |
| 93 | + in_dim = interaction_size |
| 94 | + for i, out_dim in enumerate(top_mlp_dims): |
| 95 | + top_layers.append(nn.Linear(in_dim, out_dim)) |
| 96 | + if i < len(top_mlp_dims) - 1: |
| 97 | + top_layers.append(nn.ReLU()) |
| 98 | + in_dim = out_dim |
| 99 | + self.top_mlp = nn.Sequential(*top_layers) |
| 100 | + |
| 101 | + def forward(self, dense: torch.Tensor, sparse: torch.Tensor) -> torch.Tensor: |
| 102 | + # Bottom MLP |
| 103 | + d = self.bottom_mlp(dense) |
| 104 | + |
| 105 | + # Embedding lookups |
| 106 | + embs = [emb(sparse[:, i].unsqueeze(1)) for i, emb in enumerate(self.embeddings)] |
| 107 | + |
| 108 | + # Feature interaction (dot product) |
| 109 | + combined = torch.cat([d.unsqueeze(1)] + [e.unsqueeze(1) for e in embs], dim=1) |
| 110 | + interact = torch.bmm(combined, combined.transpose(1, 2)) |
| 111 | + n = combined.size(1) |
| 112 | + idx = torch.triu_indices(n, n, offset=1) |
| 113 | + flat = interact[:, idx[0], idx[1]] |
| 114 | + |
| 115 | + # Top MLP |
| 116 | + x = torch.cat([d, flat], dim=1) |
| 117 | + return torch.sigmoid(self.top_mlp(x)) |
| 118 | + |
| 119 | + |
| 120 | +def generate_model(output_path: str, **kwargs): |
| 121 | + """Generate and save a TorchScript DLRM model.""" |
| 122 | + model = DLRM(**kwargs) |
| 123 | + param_bytes = sum(p.numel() * p.element_size() for p in model.parameters()) |
| 124 | + print(f" Parameters: {sum(p.numel() for p in model.parameters()):,}") |
| 125 | + print(f" Model size: {param_bytes / 1e6:.0f} MB") |
| 126 | + |
| 127 | + scripted = torch.jit.script(model) |
| 128 | + scripted.save(output_path) |
| 129 | + file_size = os.path.getsize(output_path) |
| 130 | + print(f" Saved to: {output_path} ({file_size / 1e6:.0f} MB on disk)") |
| 131 | + |
| 132 | + |
| 133 | +def main(): |
| 134 | + if len(sys.argv) < 2: |
| 135 | + print(f"Usage: {sys.argv[0]} <output_dir>") |
| 136 | + sys.exit(1) |
| 137 | + |
| 138 | + output_dir = sys.argv[1] |
| 139 | + os.makedirs(output_dir, exist_ok=True) |
| 140 | + |
| 141 | + # Medium model: larger embeddings (~500MB) |
| 142 | + medium_path = os.path.join(output_dir, "dlrm_medium.pt") |
| 143 | + if not os.path.exists(medium_path): |
| 144 | + print("Generating DLRM medium model...") |
| 145 | + generate_model( |
| 146 | + medium_path, |
| 147 | + emb_dim=64, |
| 148 | + max_emb_rows=500000, |
| 149 | + bottom_mlp_dims=[256, 128, 64], |
| 150 | + top_mlp_dims=[256, 128, 1], |
| 151 | + ) |
| 152 | + else: |
| 153 | + print(f"[SKIPPED] {medium_path} already exists") |
| 154 | + |
| 155 | + # Large model: even larger embeddings (~1GB) |
| 156 | + large_path = os.path.join(output_dir, "dlrm_large.pt") |
| 157 | + if not os.path.exists(large_path): |
| 158 | + print("Generating DLRM large model...") |
| 159 | + generate_model( |
| 160 | + large_path, |
| 161 | + emb_dim=128, |
| 162 | + max_emb_rows=500000, |
| 163 | + bottom_mlp_dims=[512, 256, 128], |
| 164 | + top_mlp_dims=[512, 256, 1], |
| 165 | + ) |
| 166 | + else: |
| 167 | + print(f"[SKIPPED] {large_path} already exists") |
| 168 | + |
| 169 | + |
| 170 | +if __name__ == "__main__": |
| 171 | + main() |
![meta-codesync[bot]](https://avatars.githubusercontent.com/in/1373969?v=4&size=40){kind=avatar}
0 commit comments