About testing on ImageNet1K using the finetuned weight

[study0098]

Hi! Thanks for your great work and code release. I want to test your finetuned model on some custom datasets, so I need to extract the weights and load the model in PyTorch style instead of the wrapped mmengine style. However, after extraction I got a test accuracy of 80.66 on ImageNet1K, which differs from the ~84.7 reported in the paper. Here is the code:

import argparse
import os
import os.path as osp
from copy import deepcopy

import mmengine
from mmengine.config import Config, ConfigDict, DictAction
from mmengine.evaluator import DumpResults
from mmengine.registry import RUNNERS
from mmengine.runner import Runner

import torch
import torch.nn as nn
import torch.nn.functional as F

import torchvision
import torchvision.transforms as transforms
from torchvision.transforms import InterpolationMode
import torchvision.datasets as datasets
from tqdm import tqdm
import json

from torchvision.transforms import InterpolationMode

from collections import OrderedDict

args = parse_args()

if args.out is None and args.out_item is not None:
    raise ValueError('Please use `--out` argument to specify the '
                        'path of the output file before using `--out-item`.')

cfg = Config.fromfile(args.config)

cfg = merge_args(cfg, args)

if 'runner_type' not in cfg:
    # build the default runner
    runner = Runner.from_cfg(cfg)
else:
    runner = RUNNERS.build(cfg)

if args.out and args.out_item in ['pred', None]:
    runner.test_evaluator.metrics.append(
        DumpResults(out_file_path=args.out))

mm_mean = [123.675, 116.28, 103.53]   # example (0–255 scale, MM default)
mm_std  = [58.395, 57.12, 57.375]

mean_01 = [m / 255.0 for m in mm_mean]
std_01  = [s / 255.0 for s in mm_std]


test_transform = transforms.Compose([
    # LoadImageNetFromFile -> handled by the Dataset (e.g., ImageFolder)
    transforms.Resize(256, interpolation=InterpolationMode.BICUBIC),  # ResizeEdge(short=256)
    transforms.CenterCrop(224),
    transforms.ToTensor(),                 # converts to float in [0,1]
    transforms.Normalize(mean=mean_01, std=std_01),  # matches MMCV’s pre-tensor normalize
    # Collect(keys=['img','label']) -> Dataset/DataLoader handle this
    # ToTensor(keys=['img','label']) -> image done above; labels are ints and
    # default collate will turn a list of ints into a tensor batch automatically
])
dataset_val = datasets.ImageFolder(os.path.join("/Dataset/ImageNet/", 'val'), transform=test_transform)

with open("/Dataset/ImageNet/ImageNet_val.json", "r") as f:
    items = json.load(f)
new_label_by_key = {}
for it in items:
    key = (it["prefix"], it["filename"])  # e.g., ("val/n02115641", "ILSVRC2012_val_00004334.JPEG")
    new_label_by_key[key] = int(it["label"])

new_samples, new_targets = [], []
missing = 0
for path, _orig_cls in dataset_val.samples:
    wnid = os.path.basename(os.path.dirname(path))     # e.g., "n02115641"
    fname = os.path.basename(path)                     # e.g., "ILSVRC2012_val_00004334.JPEG"
    key = (f"val/{wnid}", fname)

    if key not in new_label_by_key:
        missing += 1
        new_lbl = _orig_cls
    else:
        new_lbl = new_label_by_key[key]

    new_samples.append((path, new_lbl))
    new_targets.append(new_lbl)

dataset_val.samples = new_samples
dataset_val.targets = new_targets
if hasattr(dataset_val, "imgs"):
    dataset_val.imgs = new_samples

if missing:
    print(f"[warn] {missing} images not found in JSON; kept original labels.")

data_loader_val = torch.utils.data.DataLoader(
    dataset_val, batch_size=100, num_workers=8, pin_memory=True, drop_last=False
)

model = runner.model.module
model.eval()
model = nn.Sequential(model.backbone, model.head)
checkpoint = torch.load('cmae_base_pre1600_8x128_100e_in1k_amp_fine.pth', map_location="cpu")
state_dict = checkpoint['state_dict']
new_state_dict = OrderedDict()
for k, v in state_dict.items():
    if(k.startswith("backbone.")):
        new_state_dict[f"0.{k[9:]}"] = v
    elif(k.startswith("head.")):
        new_state_dict[f"1.{k[5:]}"] = v
model.load_state_dict(new_state_dict, strict=True)
model.eval()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

correct = 0
total = 0
with torch.no_grad():
    for images, labels in tqdm(data_loader_val):
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)[0]
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
print(f'Accuracy of the model on the validation images: {100 * correct / total} %')

metrics = runner.test()

The code is missing some env setting as I didn't copy everything. As in the code, I changed the label ordering to match the one given in the ImageNet_val.json and load the model accordingly with strict weight loading protocol. The code gives:

Accuracy of the model on the validation images: 80.662 %
09/19 14:57:42 - mmengine - WARNING - Dataset ImageNetDataset has no metainfo. ``dataset_meta`` in evaluator, metric and visualizer will be None.
Loads checkpoint by local backend from path: cmae_base_pre1600_8x128_100e_in1k_amp_fine.pth
09/19 14:57:42 - mmengine - INFO - Load checkpoint from cmae_base_pre1600_8x128_100e_in1k_amp_fine.pth
09/19 14:57:43 - mmengine - WARNING - Dataset ImageNetDataset has no metainfo. ``dataset_meta`` in visualizer will be None.
09/19 14:57:43 - mmengine - INFO - resumed epoch: 100, iter: 125200
09/19 14:57:52 - mmengine - INFO - Epoch(test) [ 50/391]    eta: 0:00:58  time: 0.1325  data_time: 0.0465  memory: 1624  
09/19 14:58:01 - mmengine - INFO - Epoch(test) [100/391]    eta: 0:00:50  time: 0.2371  data_time: 0.1413  memory: 1624  
09/19 14:58:12 - mmengine - INFO - Epoch(test) [150/391]    eta: 0:00:45  time: 0.1453  data_time: 0.0587  memory: 1624  
09/19 14:58:19 - mmengine - INFO - Epoch(test) [200/391]    eta: 0:00:34  time: 0.1876  data_time: 0.1016  memory: 1624  
09/19 14:58:27 - mmengine - INFO - Epoch(test) [250/391]    eta: 0:00:24  time: 0.1264  data_time: 0.0397  memory: 1624  
09/19 14:58:33 - mmengine - INFO - Epoch(test) [300/391]    eta: 0:00:15  time: 0.1117  data_time: 0.0242  memory: 1624  
09/19 14:58:41 - mmengine - INFO - Epoch(test) [350/391]    eta: 0:00:06  time: 0.1548  data_time: 0.0674  memory: 1624  
09/19 14:58:47 - mmengine - INFO - Epoch(test) [391/391]    accuracy/top1: 84.6140  accuracy/top5: 97.0960  data_time: 0.0675  time: 0.1607

There is a difference between the two tests. I wonder what caused this difference, could it be the data transform or the model loading? This might hugely affect a straightforward downstream testing of the pretrained models. Could you please check out the code? Thanks in advance for your time on this!