Lecture 8 - Conformalized Classification

Much of this notebook is from https://github.com/aangelopoulos/conformal-prediction/blob/main/notebooks/imagenet-aps.ipynb

import os
import json
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.image import imread
!pip install -U --no-cache-dir gdown --pre

Collecting gdown
  Downloading gdown-5.2.0-py3-none-any.whl.metadata (5.8 kB)
Requirement already satisfied: beautifulsoup4 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from gdown) (4.13.5)
Requirement already satisfied: filelock in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from gdown) (3.19.1)
Requirement already satisfied: requests[socks] in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from gdown) (2.32.5)
Requirement already satisfied: tqdm in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from gdown) (4.67.1)
Requirement already satisfied: soupsieve>1.2 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from beautifulsoup4->gdown) (2.5)
Requirement already satisfied: typing-extensions>=4.0.0 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from beautifulsoup4->gdown) (4.15.0)
Requirement already satisfied: charset_normalizer<4,>=2 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from requests[socks]->gdown) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from requests[socks]->gdown) (3.7)
Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from requests[socks]->gdown) (2.5.0)
Requirement already satisfied: certifi>=2017.4.17 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from requests[socks]->gdown) (2025.8.3)
Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /Users/gm845/anaconda3/envs/msds534/lib/python3.13/site-packages (from requests[socks]->gdown) (1.7.1)
Downloading gdown-5.2.0-py3-none-any.whl (18 kB)
Installing collected packages: gdown
Successfully installed gdown-5.2.0

# Load cached data
if not os.path.exists("../data/imagenet"):
    os.system("gdown 1h7S6N_Rx7gdfO3ZunzErZy6H7620EbZK -O ../data.tar.gz")
    os.system("tar -xf ../data.tar.gz -C ../")
    os.system("rm ../data.tar.gz")
if not os.path.exists("../data/imagenet/human_readable_labels.json"):
    from urllib.request import urlretrieve
    urlretrieve("https://raw.githubusercontent.com/anishathalye/imagenet-simple-labels/master/imagenet-simple-labels.json","../data/imagenet/human_readable_labels.json")

data = np.load("../data/imagenet/imagenet-resnet152.npz")
example_paths = os.listdir("../data/imagenet/examples")
smx = data["smx"]
labels = data["labels"].astype(int)

Downloading...
From (original): https://drive.google.com/uc?id=1h7S6N_Rx7gdfO3ZunzErZy6H7620EbZK
From (redirected): https://drive.google.com/uc?id=1h7S6N_Rx7gdfO3ZunzErZy6H7620EbZK&confirm=t&uuid=bfa0478f-2f18-47b2-bc1d-b5064f0a2768
To: /Users/gm845/Library/CloudStorage/Box-Box/teaching/2025/msds-534/website/data.tar.gz
100%|█████████████████████████████████████| 1.31G/1.31G [00:16<00:00, 77.9MB/s]

smx.shape

(50000, 1000)

# Problem setup
n = 1000  # number of calibration points
alpha = 0.1  # 1-alpha is the desired coverage (0.9 coverage)

# Split the softmax scores into calibration and validation sets (save the shuffling)
idx = np.array([1] * n + [0] * (smx.shape[0] - n)) > 0
np.random.shuffle(idx)
cal_smx, val_smx = smx[idx, :], smx[~idx, :]
cal_labels, val_labels = labels[idx], labels[~idx]

# get indices of sorted softmax (ascending)
cal_pi = cal_smx.argsort(axis=1)

# reverse, so descending
cal_pi = cal_pi[:, ::-1] 

# order the softmax scores 
cal_srt = np.take_along_axis(cal_smx, cal_pi, axis=1)

# take cumulative sum
cal_srt = cal_srt.cumsum(axis=1)

cal_srt.shape

(1000, 1000)

# applying argsort to cal_pi gets the rank - we reorder cal_srt to original space

cal_scores = np.take_along_axis(cal_srt, cal_pi.argsort(axis=1), axis=1)[
    range(n), cal_labels
]

# Get the score quantile
qhat = np.quantile(
    cal_scores, np.ceil((n + 1) * (1 - alpha)) / n, interpolation="higher"
)
# Deploy (output=list of length n, each element is tensor of classes)
val_pi = val_smx.argsort(1)[:, ::-1]
val_srt = np.take_along_axis(val_smx, val_pi, axis=1).cumsum(axis=1)
prediction_sets = np.take_along_axis(val_srt <= qhat, val_pi.argsort(axis=1), axis=1)

# Calculate empirical coverage
empirical_coverage = prediction_sets[
    np.arange(prediction_sets.shape[0]), val_labels
].mean()
print(f"The empirical coverage is: {empirical_coverage}")

The empirical coverage is: 0.8913877551020408

# Show some examples
with open("../data/imagenet/human_readable_labels.json") as f:
    label_strings = np.array(json.load(f))

example_paths = os.listdir("../data/imagenet/examples")
for i in range(10):
    rand_path = np.random.choice(example_paths)
    img = imread("../data/imagenet/examples/" + rand_path)
    img_index = int(rand_path.split(".")[0])
    img_pi = smx[img_index].argsort()[::-1]
    img_srt = np.take_along_axis(smx[img_index], img_pi, axis=0).cumsum()
    prediction_set = np.take_along_axis(img_srt <= qhat, img_pi.argsort(), axis=0)
    plt.figure()
    plt.imshow(img)
    plt.axis("off")
    plt.show()
    print(f"The prediction set is: {list(label_strings[prediction_set])}")

The prediction set is: ['great white shark', 'hen', 'bulbul', 'jay', 'chickadee', 'American dipper', 'leatherback sea turtle', 'mud turtle', 'Nile crocodile', 'American alligator', 'triceratops', 'eastern hog-nosed snake', 'Saharan horned viper', 'southern black widow', 'tick', 'grey parrot', 'wallaby', 'wombat', 'Dungeness crab', 'American lobster', 'spiny lobster', 'crayfish', 'common gallinule', 'Chihuahua', 'toy terrier', 'Rhodesian Ridgeback', 'Afghan Hound', 'Bloodhound', 'Bluetick Coonhound', 'Black and Tan Coonhound', 'Redbone Coonhound', 'Irish Wolfhound', 'Italian Greyhound', 'Whippet', 'Ibizan Hound', 'Norwegian Elkhound', 'Saluki', 'Scottish Deerhound', 'Weimaraner', 'Staffordshire Bull Terrier', 'American Staffordshire Terrier', 'Bedlington Terrier', 'Border Terrier', 'Kerry Blue Terrier', 'Irish Terrier', 'Norfolk Terrier', 'Norwich Terrier', 'Wire Fox Terrier', 'Lakeland Terrier', 'Airedale Terrier', 'Cairn Terrier', 'Australian Terrier', 'Boston Terrier', 'Miniature Schnauzer', 'Giant Schnauzer', 'Standard Schnauzer', 'Scottish Terrier', 'Australian Silky Terrier', 'Soft-coated Wheaten Terrier', 'West Highland White Terrier', 'Flat-Coated Retriever', 'Labrador Retriever', 'Chesapeake Bay Retriever', 'German Shorthaired Pointer', 'Kuvasz', 'Schipperke', 'Groenendael', 'Malinois', 'Briard', 'Australian Kelpie', 'Shetland Sheepdog', 'collie', 'Border Collie', 'Bouvier des Flandres', 'Rottweiler', 'German Shepherd Dog', 'Dobermann', 'Miniature Pinscher', 'Greater Swiss Mountain Dog', 'Appenzeller Sennenhund', 'Boxer', 'Bullmastiff', 'Tibetan Mastiff', 'French Bulldog', 'Great Dane', 'St. Bernard', 'husky', 'Alaskan Malamute', 'Siberian Husky', 'Dalmatian', 'Basenji', 'pug', 'Leonberger', 'Chow Chow', 'Keeshond', 'Griffon Bruxellois', 'Pembroke Welsh Corgi', 'Cardigan Welsh Corgi', 'Toy Poodle', 'Standard Poodle', 'Mexican hairless dog', 'grey wolf', 'Alaskan tundra wolf', 'red wolf', 'coyote', 'dingo', 'dhole', 'African wild dog', 'red fox', 'kit fox', 'Arctic fox', 'grey fox', 'Siamese cat', 'Egyptian Mau', 'cougar', 'lynx', 'ground beetle', 'longhorn beetle', 'rhinoceros beetle', 'weevil', 'grasshopper', 'cricket', 'cockroach', 'starfish', 'hare', 'weasel', 'black-footed ferret', 'badger', 'armadillo', 'baboon', 'snoek', 'coho salmon', 'garfish', 'accordion', 'apron', 'backpack', 'ballpoint pen', 'baseball', 'bath towel', 'bib', 'bolo tie', 'poke bonnet', 'bow tie', 'bra', 'bucket', 'buckle', 'bulletproof vest', 'cauldron', 'canoe', 'cardigan', 'tool kit', 'carton', 'automated teller machine', 'catamaran', 'mobile phone', 'chain', 'chain-link fence', 'chainsaw', 'Christmas stocking', 'cleaver', 'clogs', 'coffee mug', 'coffeemaker', 'coil', 'combination lock', 'computer keyboard', 'cowboy hat', 'digital watch', 'dishcloth', 'dishwasher', 'dog sled', 'doormat', 'drum', 'drumstick', 'fountain pen', 'fur coat', 'grand piano', 'hammer', 'hair dryer', 'hand-held computer', 'handkerchief', 'harvester', 'hatchet', 'holster', 'horse-drawn vehicle', 'iPod', 'clothes iron', 'jeans', 'jeep', 'T-shirt', 'jigsaw puzzle', 'joystick', 'kimono', 'knee pad', 'knot', 'ladle', 'lawn mower', 'lens cap', 'lighter', 'slip-on shoe', 'magnetic compass', 'mail bag', 'tights', 'mask', 'match', 'microphone', 'military uniform', 'mitten', 'mortar', 'mousetrap', 'muzzle', 'nail', 'oscilloscope', 'packet', 'pan flute', 'paper towel', 'pencil sharpener', 'Pickelhaube', 'ping-pong ball', 'plastic bag', 'plunger', 'Polaroid camera', 'pole', 'police van', 'power drill', 'purse', 'quilt', 'racket', 'radiator', 'reel', 'reflex camera', 'remote control', 'revolver', 'eraser', 'running shoe', 'sandal', 'scabbard', 'schooner', 'screw', 'screwdriver', 'seat belt', 'shield', 'shoe store', 'shoji', 'shopping basket', 'shopping cart', 'shovel', 'shower cap', 'ski mask', 'sleeping bag', 'sliding door', 'soccer ball', 'solar thermal collector', 'sombrero', 'soup bowl', 'space heater', 'spatula', 'spindle', 'spotlight', 'scarf', 'stopwatch', 'stove', 'strainer', 'suit', 'sunglass', 'sunglasses', 'mop', 'sweatshirt', 'syringe', 'television', 'tennis ball', 'trench coat', 'tripod', 'tub', 'vacuum cleaner', 'sink', 'water bottle', 'whistle', 'window screen', 'Windsor tie', 'wok', 'wool', 'split-rail fence', 'crossword', 'dust jacket', 'plate', 'guacamole', 'ice pop', 'pretzel', 'broccoli', 'zucchini', 'orange', 'lemon', 'pineapple', 'pizza', 'corn', 'acorn', 'rose hip', 'ear', 'toilet paper']

The prediction set is: ['tennis ball']

The prediction set is: ['great white shark', 'tiger shark', 'hammerhead shark', 'electric ray', 'stingray', 'cock', 'hen', 'ostrich', 'brambling', 'goldfinch', 'house finch', 'junco', 'indigo bunting', 'American robin', 'bulbul', 'jay', 'magpie', 'chickadee', 'American dipper', 'kite', 'bald eagle', 'vulture', 'great grey owl', 'axolotl', 'American bullfrog', 'tree frog', 'tailed frog', 'loggerhead sea turtle', 'leatherback sea turtle', 'mud turtle', 'terrapin', 'box turtle', 'banded gecko', 'green iguana', 'Carolina anole', 'desert grassland whiptail lizard', 'agama', 'frilled-necked lizard', 'alligator lizard', 'European green lizard', 'chameleon', 'Komodo dragon', 'American alligator', 'triceratops', 'worm snake', 'eastern hog-nosed snake', 'smooth green snake', 'garter snake', 'water snake', 'vine snake', 'night snake', 'boa constrictor', 'African rock python', 'Indian cobra', 'green mamba', 'Saharan horned viper', 'eastern diamondback rattlesnake', 'harvestman', 'yellow garden spider', 'barn spider', 'European garden spider', 'southern black widow', 'tarantula', 'wolf spider', 'tick', 'centipede', 'ruffed grouse', 'prairie grouse', 'peacock', 'quail', 'partridge', 'grey parrot', 'macaw', 'sulphur-crested cockatoo', 'lorikeet', 'coucal', 'bee eater', 'hornbill', 'hummingbird', 'jacamar', 'toucan', 'duck', 'goose', 'tusker', 'wallaby', 'koala', 'wombat', 'jellyfish', 'sea anemone', 'nematode', 'conch', 'snail', 'slug', 'chambered nautilus', 'Dungeness crab', 'rock crab', 'crayfish', 'white stork', 'black stork', 'spoonbill', 'flamingo', 'little blue heron', 'great egret', 'bittern', 'crane (bird)', 'limpkin', 'common gallinule', 'American coot', 'bustard', 'dunlin', 'dowitcher', 'pelican', 'king penguin', 'albatross', 'killer whale', 'dugong', 'sea lion', 'Chihuahua', 'Japanese Chin', 'Maltese', 'Shih Tzu', 'King Charles Spaniel', 'Papillon', 'toy terrier', 'Rhodesian Ridgeback', 'Afghan Hound', 'Basset Hound', 'Beagle', 'Bloodhound', 'Bluetick Coonhound', 'Black and Tan Coonhound', 'Redbone Coonhound', 'Irish Wolfhound', 'Italian Greyhound', 'Whippet', 'Ibizan Hound', 'Norwegian Elkhound', 'Otterhound', 'Saluki', 'Scottish Deerhound', 'Weimaraner', 'Staffordshire Bull Terrier', 'American Staffordshire Terrier', 'Bedlington Terrier', 'Border Terrier', 'Kerry Blue Terrier', 'Irish Terrier', 'Norfolk Terrier', 'Norwich Terrier', 'Yorkshire Terrier', 'Wire Fox Terrier', 'Lakeland Terrier', 'Sealyham Terrier', 'Airedale Terrier', 'Cairn Terrier', 'Australian Terrier', 'Dandie Dinmont Terrier', 'Boston Terrier', 'Miniature Schnauzer', 'Giant Schnauzer', 'Standard Schnauzer', 'Tibetan Terrier', 'Australian Silky Terrier', 'Soft-coated Wheaten Terrier', 'West Highland White Terrier', 'Lhasa Apso', 'Flat-Coated Retriever', 'Curly-coated Retriever', 'Golden Retriever', 'Labrador Retriever', 'Chesapeake Bay Retriever', 'German Shorthaired Pointer', 'Vizsla', 'English Setter', 'Irish Setter', 'Gordon Setter', 'Clumber Spaniel', 'English Springer Spaniel', 'Welsh Springer Spaniel', 'Cocker Spaniels', 'Sussex Spaniel', 'Irish Water Spaniel', 'Kuvasz', 'Schipperke', 'Groenendael', 'Malinois', 'Briard', 'Australian Kelpie', 'Komondor', 'Old English Sheepdog', 'Shetland Sheepdog', 'collie', 'Bouvier des Flandres', 'Rottweiler', 'German Shepherd Dog', 'Dobermann', 'Miniature Pinscher', 'Greater Swiss Mountain Dog', 'Bernese Mountain Dog', 'Entlebucher Sennenhund', 'Boxer', 'Bullmastiff', 'Tibetan Mastiff', 'French Bulldog', 'Great Dane', 'St. Bernard', 'husky', 'Alaskan Malamute', 'Siberian Husky', 'Dalmatian', 'Affenpinscher', 'Basenji', 'pug', 'Newfoundland', 'Pyrenean Mountain Dog', 'Samoyed', 'Pomeranian', 'Chow Chow', 'Keeshond', 'Toy Poodle', 'Miniature Poodle', 'Standard Poodle', 'Mexican hairless dog', 'red wolf', 'dingo', 'African wild dog', 'red fox', 'Arctic fox', 'grey fox', 'tabby cat', 'tiger cat', 'Persian cat', 'Siamese cat', 'Egyptian Mau', 'cougar', 'lynx', 'leopard', 'snow leopard', 'jaguar', 'tiger', 'brown bear', 'American black bear', 'polar bear', 'sloth bear', 'mongoose', 'meerkat', 'ladybug', 'ground beetle', 'longhorn beetle', 'leaf beetle', 'dung beetle', 'rhinoceros beetle', 'weevil', 'fly', 'bee', 'ant', 'grasshopper', 'cricket', 'stick insect', 'mantis', 'cicada', 'leafhopper', 'lacewing', 'dragonfly', 'damselfly', 'red admiral', 'ringlet', 'monarch butterfly', 'small white', 'sulphur butterfly', 'gossamer-winged butterfly', 'starfish', 'cottontail rabbit', 'hare', 'Angora rabbit', 'hamster', 'porcupine', 'fox squirrel', 'marmot', 'beaver', 'common sorrel', 'zebra', 'pig', 'wild boar', 'hippopotamus', 'ox', 'water buffalo', 'Alpine ibex', 'impala', 'gazelle', 'dromedary', 'llama', 'weasel', 'mink', 'European polecat', 'black-footed ferret', 'otter', 'skunk', 'badger', 'armadillo', 'three-toed sloth', 'orangutan', 'gorilla', 'chimpanzee', 'gibbon', 'siamang', 'guenon', 'patas monkey', 'baboon', 'macaque', 'langur', 'black-and-white colobus', 'proboscis monkey', 'marmoset', 'white-headed capuchin', 'howler monkey', 'titi', "Geoffroy's spider monkey", 'common squirrel monkey', 'ring-tailed lemur', 'indri', 'Asian elephant', 'African bush elephant', 'red panda', 'giant panda', 'clownfish', 'abacus', 'abaya', 'accordion', 'acoustic guitar', 'aircraft carrier', 'airliner', 'airship', 'ambulance', 'amphibious vehicle', 'analog clock', 'apiary', 'waste container', 'assault rifle', 'backpack', 'bakery', 'balloon', 'Band-Aid', 'banjo', 'baluster', 'barbell', 'barber chair', 'barbershop', 'barn', 'barometer', 'barrel', 'wheelbarrow', 'baseball', 'basketball', 'bassinet', 'bassoon', 'swimming cap', 'bath towel', 'bathtub', 'station wagon', 'lighthouse', 'military cap', 'bell-cot', 'bib', 'bikini', 'binoculars', 'birdhouse', 'boathouse', 'poke bonnet', 'bookcase', 'bookstore', 'bow tie', 'brass', 'bra', 'breakwater', 'breastplate', 'broom', 'bucket', 'bulletproof vest', 'high-speed train', 'taxicab', 'cauldron', 'candle', 'cannon', 'canoe', 'can opener', 'car mirror', 'carousel', 'carton', 'car wheel', 'automated teller machine', 'cassette player', 'castle', 'catamaran', 'CD player', 'cello', 'mobile phone', 'chain', 'chain-link fence', 'chainsaw', 'chest', 'chiffonier', 'chime', 'Christmas stocking', 'church', 'movie theater', 'cleaver', 'cloak', 'clogs', 'cocktail shaker', 'coffee mug', 'coffeemaker', 'coil', 'computer keyboard', 'confectionery store', 'convertible', 'cowboy hat', 'cradle', 'crane (machine)', 'crash helmet', 'crate', 'infant bed', 'Crock Pot', 'croquet ball', 'crutch', 'dam', 'desk', 'desktop computer', 'rotary dial telephone', 'diaper', 'digital clock', 'digital watch', 'dining table', 'dishcloth', 'dishwasher', 'disc brake', 'dock', 'dome', 'doormat', 'drilling rig', 'drum', 'drumstick', 'dumbbell', 'Dutch oven', 'electric fan', 'electric locomotive', 'entertainment center', 'envelope', 'filing cabinet', 'fireboat', 'fire engine', 'fire screen sheet', 'flagpole', 'flute', 'folding chair', 'football helmet', 'forklift', 'fountain', 'four-poster bed', 'French horn', 'frying pan', 'garbage truck', 'gas mask', 'gas pump', 'goblet', 'golf ball', 'golf cart', 'gondola', 'gong', 'gown', 'grand piano', 'greenhouse', 'grille', 'grocery store', 'guillotine', 'barrette', 'half-track', 'hamper', 'hand-held computer', 'handkerchief', 'harmonica', 'harvester', 'home theater', 'honeycomb', 'hook', 'hoop skirt', 'horse-drawn vehicle', "jack-o'-lantern", 'jeans', 'jeep', 'T-shirt', 'pulled rickshaw', 'joystick', 'kimono', 'knee pad', 'knot', 'lab coat', 'ladle', 'lampshade', 'lawn mower', 'library', 'lifeboat', 'lighter', 'limousine', 'ocean liner', 'lipstick', 'lotion', 'speaker', 'loupe', 'sawmill', 'magnetic compass', 'mail bag', 'mailbox', 'tights', 'tank suit', 'maraca', 'marimba', 'mask', 'maypole', 'maze', 'measuring cup', 'microphone', 'military uniform', 'milk can', 'minibus', 'miniskirt', 'minivan', 'missile', 'mitten', 'mixing bowl', 'mobile home', 'Model T', 'monastery', 'monitor', 'moped', 'mortar', 'square academic cap', 'mosque', 'mosquito net', 'scooter', 'mountain bike', 'computer mouse', 'mousetrap', 'moving van', 'neck brace', 'necklace', 'nipple', 'obelisk', 'oboe', 'organ', 'oscilloscope', 'bullock cart', 'oxygen mask', 'packet', 'paddle', 'pajamas', 'palace', 'pan flute', 'paper towel', 'parachute', 'park bench', 'parking meter', 'patio', 'payphone', 'pedestal', 'perfume', 'Petri dish', 'photocopier', 'Pickelhaube', 'picket fence', 'pickup truck', 'pier', 'piggy bank', 'pillow', 'ping-pong ball', 'pinwheel', 'pitcher', 'planetarium', 'plastic bag', 'plow', 'plunger', 'pole', 'police van', 'poncho', 'billiard table', 'soda bottle', 'pot', "potter's wheel", 'power drill', 'prayer rug', 'printer', 'prison', 'projectile', 'punching bag', 'purse', 'quilt', 'racket', 'radiator', 'radio', 'radio telescope', 'rain barrel', 'recreational vehicle', 'reflex camera', 'refrigerator', 'restaurant', 'rifle', 'rocking chair', 'eraser', 'rugby ball', 'ruler', 'safe', 'salt shaker', 'sandal', 'sarong', 'saxophone', 'weighing scale', 'school bus', 'schooner', 'scoreboard', 'CRT screen', 'screw', 'seat belt', 'sewing machine', 'shoe store', 'shoji', 'shopping basket', 'shopping cart', 'shovel', 'shower cap', 'shower curtain', 'ski mask', 'sleeping bag', 'slide rule', 'sliding door', 'soap dispenser', 'soccer ball', 'sock', 'solar thermal collector', 'sombrero', 'soup bowl', 'space bar', 'space heater', 'space shuttle', 'spatula', 'motorboat', 'spider web', 'spotlight', 'steam locomotive', 'through arch bridge', 'stethoscope', 'scarf', 'stone wall', 'stove', 'strainer', 'tram', 'stretcher', 'couch', 'stupa', 'submarine', 'suit', 'sundial', 'sunglass', 'sunglasses', 'sunscreen', 'suspension bridge', 'mop', 'sweatshirt', 'swimsuit', 'swing', 'syringe', 'table lamp', 'tank', 'tape player', 'teapot', 'teddy bear', 'television', 'tennis ball', 'thatched roof', 'front curtain', 'threshing machine', 'tile roof', 'toaster', 'tobacco shop', 'totem pole', 'tow truck', 'toy store', 'tractor', 'tray', 'tricycle', 'trimaran', 'tripod', 'triumphal arch', 'trolleybus', 'tub', 'turnstile', 'typewriter keyboard', 'umbrella', 'unicycle', 'upright piano', 'vase', 'vault', 'velvet', 'vending machine', 'vestment', 'viaduct', 'volleyball', 'wall clock', 'wardrobe', 'military aircraft', 'sink', 'washing machine', 'water bottle', 'water jug', 'water tower', 'whiskey jug', 'whistle', 'wig', 'window screen', 'window shade', 'wine bottle', 'wok', 'wooden spoon', 'wool', 'split-rail fence', 'shipwreck', 'yurt', 'website', 'comic book', 'crossword', 'traffic sign', 'traffic light', 'dust jacket', 'menu', 'plate', 'guacamole', 'consomme', 'hot pot', 'ice pop', 'baguette', 'bagel', 'pretzel', 'cheeseburger', 'hot dog', 'mashed potato', 'cabbage', 'broccoli', 'cauliflower', 'zucchini', 'spaghetti squash', 'acorn squash', 'butternut squash', 'cucumber', 'artichoke', 'bell pepper', 'cardoon', 'mushroom', 'Granny Smith', 'strawberry', 'orange', 'lemon', 'fig', 'pineapple', 'banana', 'jackfruit', 'custard apple', 'pomegranate', 'hay', 'carbonara', 'dough', 'meatloaf', 'pot pie', 'burrito', 'red wine', 'cup', 'eggnog', 'alp', 'bubble', 'cliff', 'coral reef', 'lakeshore', 'promontory', 'shoal', 'seashore', 'valley', 'volcano', 'bridegroom', 'scuba diver', 'rapeseed', 'daisy', "yellow lady's slipper", 'corn', 'acorn', 'rose hip', 'horse chestnut seed', 'agaric', 'gyromitra', 'stinkhorn mushroom', 'earth star', 'hen-of-the-woods', 'bolete', 'ear']

The prediction set is: ['accordion', 'banjo', 'bassoon', 'cornet', 'flute', 'French horn', 'microphone', 'oboe', 'saxophone', 'stage', 'trombone']

The prediction set is: ['jay', 'agama', 'trilobite', 'tick', 'centipede', 'grey parrot', 'macaw', 'slug', 'fiddler crab', 'common gallinule', 'Pekingese', 'Shih Tzu', 'Papillon', 'toy terrier', 'Redbone Coonhound', 'West Highland White Terrier', 'Schipperke', 'Groenendael', 'Shetland Sheepdog', 'collie', 'Entlebucher Sennenhund', 'Samoyed', 'Pomeranian', 'Keeshond', 'Pembroke Welsh Corgi', 'tabby cat', 'tiger cat', 'Egyptian Mau', 'ground beetle', 'dung beetle', 'rhinoceros beetle', 'leafhopper', 'dragonfly', 'damselfly', 'sea urchin', 'armadillo', 'garfish', 'abacus', 'accordion', 'analog clock', 'backpack', 'ballpoint pen', 'Band-Aid', 'banjo', 'baseball', 'bassoon', 'beaker', 'bib', 'ring binder', 'bolo tie', 'bookcase', 'bookstore', 'bow', 'bow tie', 'broom', 'bucket', 'buckle', 'cauldron', 'candle', 'can opener', 'tool kit', 'carton', 'cassette', 'cassette player', 'CD player', 'mobile phone', 'chain', 'chime', 'Christmas stocking', 'cleaver', 'cloak', 'coffee mug', 'coil', 'computer keyboard', 'confectionery store', 'corkscrew', 'croquet ball', 'crutch', 'desk', 'desktop computer', 'diaper', 'digital clock', 'digital watch', 'dishcloth', 'doormat', 'drum', 'drumstick', 'electric guitar', 'envelope', 'face powder', 'feather boa', 'filing cabinet', 'flute', 'fountain pen', 'grand piano', 'barrette', 'hair spray', 'hammer', 'hand-held computer', 'handkerchief', 'hard disk drive', 'harmonica', 'holster', 'hook', 'iPod', 'jeans', 'jigsaw puzzle', 'knot', 'lab coat', 'ladle', 'laptop computer', 'lens cap', 'paper knife', 'library', 'lighter', 'lipstick', 'lotion', 'loupe', 'mail bag', 'maraca', 'marimba', 'mask', 'match', 'maypole', 'medicine chest', 'microphone', 'miniskirt', 'mitten', 'modem', 'square academic cap', 'computer mouse', 'nail', 'necklace', 'nipple', 'oboe', 'ocarina', 'oil filter', 'organ', 'oscilloscope', 'packet', 'paddle', 'paintbrush', 'pan flute', 'paper towel', 'pencil case', 'pencil sharpener', 'perfume', 'plectrum', 'piggy bank', 'pill bottle', 'pinwheel', 'plastic bag', 'plunger', 'Polaroid camera', 'pole', 'pot', 'prayer rug', 'printer', 'purse', 'quill', 'racket', 'radiator', 'radio', 'reel', 'remote control', 'revolver', 'rifle', 'eraser', 'ruler', 'safety pin', 'salt shaker', 'scabbard', 'screw', 'screwdriver', 'shovel', 'ski', 'ski mask', 'slide rule', 'snorkel', 'sock', 'sombrero', 'soup bowl', 'spatula', 'spindle', 'steel drum', 'stethoscope', 'scarf', 'strainer', 'sunglass', 'sunglasses', 'sunscreen', 'mop', 'sweatshirt', 'switch', 'syringe', 'tape player', 'tennis ball', 'thimble', 'tobacco shop', 'torch', 'tray', 'tripod', 'umbrella', 'upright piano', 'vase', 'velvet', 'violin', 'wall clock', 'wallet', 'water bottle', 'whistle', 'wig', 'Windsor tie', 'wooden spoon', 'wool', 'comic book', 'crossword', 'dust jacket', 'ice pop', 'zucchini', 'Granny Smith', 'pineapple', 'cup', 'bubble', 'shoal', 'toilet paper']

The prediction set is: ['wallaby', 'hyena', 'mongoose', 'meerkat', 'marmot', 'badger', 'ring-tailed lemur']

The prediction set is: ['Japanese Chin', 'Pekingese', 'Shih Tzu', 'Lhasa Apso', 'Tibetan Mastiff', 'Affenpinscher', 'pug', 'Pomeranian', 'Keeshond', 'Griffon Bruxellois', 'Persian cat']

The prediction set is: ['tench', 'great white shark', 'tiger shark', 'hammerhead shark', 'electric ray', 'cock', 'hen', 'ostrich', 'brambling', 'magpie', 'American dipper', 'kite', 'vulture', 'great grey owl', 'fire salamander', 'smooth newt', 'newt', 'spotted salamander', 'axolotl', 'American bullfrog', 'tree frog', 'tailed frog', 'loggerhead sea turtle', 'leatherback sea turtle', 'mud turtle', 'terrapin', 'box turtle', 'banded gecko', 'green iguana', 'desert grassland whiptail lizard', 'agama', 'frilled-necked lizard', 'alligator lizard', 'Gila monster', 'chameleon', 'Komodo dragon', 'Nile crocodile', 'American alligator', 'triceratops', 'worm snake', 'ring-necked snake', 'eastern hog-nosed snake', 'kingsnake', 'garter snake', 'vine snake', 'night snake', 'boa constrictor', 'African rock python', 'Indian cobra', 'green mamba', 'sea snake', 'Saharan horned viper', 'eastern diamondback rattlesnake', 'sidewinder', 'trilobite', 'harvestman', 'scorpion', 'yellow garden spider', 'barn spider', 'tarantula', 'tick', 'centipede', 'black grouse', 'ptarmigan', 'ruffed grouse', 'prairie grouse', 'peacock', 'quail', 'partridge', 'grey parrot', 'coucal', 'hornbill', 'hummingbird', 'jacamar', 'toucan', 'red-breasted merganser', 'goose', 'black swan', 'tusker', 'echidna', 'platypus', 'wallaby', 'koala', 'wombat', 'brain coral', 'flatworm', 'nematode', 'conch', 'snail', 'slug', 'sea slug', 'chiton', 'chambered nautilus', 'Dungeness crab', 'rock crab', 'fiddler crab', 'red king crab', 'American lobster', 'isopod', 'black stork', 'great egret', 'crane (bird)', 'limpkin', 'common gallinule', 'American coot', 'bustard', 'dowitcher', 'pelican', 'king penguin', 'albatross', 'grey whale', 'killer whale', 'dugong', 'sea lion', 'Chihuahua', 'Japanese Chin', 'Maltese', 'Pekingese', 'Shih Tzu', 'King Charles Spaniel', 'Papillon', 'toy terrier', 'Rhodesian Ridgeback', 'Afghan Hound', 'Basset Hound', 'Beagle', 'Bloodhound', 'Bluetick Coonhound', 'Black and Tan Coonhound', 'Treeing Walker Coonhound', 'English foxhound', 'Redbone Coonhound', 'borzoi', 'Irish Wolfhound', 'Italian Greyhound', 'Whippet', 'Ibizan Hound', 'Norwegian Elkhound', 'Otterhound', 'Saluki', 'Scottish Deerhound', 'Weimaraner', 'Staffordshire Bull Terrier', 'American Staffordshire Terrier', 'Bedlington Terrier', 'Border Terrier', 'Kerry Blue Terrier', 'Irish Terrier', 'Norfolk Terrier', 'Norwich Terrier', 'Yorkshire Terrier', 'Wire Fox Terrier', 'Lakeland Terrier', 'Sealyham Terrier', 'Airedale Terrier', 'Cairn Terrier', 'Australian Terrier', 'Dandie Dinmont Terrier', 'Boston Terrier', 'Miniature Schnauzer', 'Giant Schnauzer', 'Standard Schnauzer', 'Scottish Terrier', 'Tibetan Terrier', 'Australian Silky Terrier', 'Soft-coated Wheaten Terrier', 'West Highland White Terrier', 'Flat-Coated Retriever', 'Curly-coated Retriever', 'Golden Retriever', 'Labrador Retriever', 'Chesapeake Bay Retriever', 'German Shorthaired Pointer', 'Vizsla', 'English Setter', 'Irish Setter', 'Gordon Setter', 'Brittany', 'Clumber Spaniel', 'English Springer Spaniel', 'Welsh Springer Spaniel', 'Cocker Spaniels', 'Sussex Spaniel', 'Irish Water Spaniel', 'Kuvasz', 'Schipperke', 'Groenendael', 'Malinois', 'Briard', 'Australian Kelpie', 'Komondor', 'Old English Sheepdog', 'Shetland Sheepdog', 'collie', 'Border Collie', 'Bouvier des Flandres', 'Rottweiler', 'German Shepherd Dog', 'Dobermann', 'Miniature Pinscher', 'Greater Swiss Mountain Dog', 'Bernese Mountain Dog', 'Appenzeller Sennenhund', 'Entlebucher Sennenhund', 'Boxer', 'Bullmastiff', 'Tibetan Mastiff', 'French Bulldog', 'Great Dane', 'St. Bernard', 'husky', 'Alaskan Malamute', 'Siberian Husky', 'Dalmatian', 'Affenpinscher', 'Basenji', 'pug', 'Leonberger', 'Newfoundland', 'Pyrenean Mountain Dog', 'Samoyed', 'Pomeranian', 'Chow Chow', 'Keeshond', 'Griffon Bruxellois', 'Pembroke Welsh Corgi', 'Cardigan Welsh Corgi', 'Toy Poodle', 'Miniature Poodle', 'Standard Poodle', 'Mexican hairless dog', 'grey wolf', 'Alaskan tundra wolf', 'red wolf', 'coyote', 'dingo', 'dhole', 'African wild dog', 'hyena', 'red fox', 'kit fox', 'Arctic fox', 'grey fox', 'tabby cat', 'tiger cat', 'Persian cat', 'Siamese cat', 'Egyptian Mau', 'cougar', 'lynx', 'leopard', 'snow leopard', 'jaguar', 'lion', 'tiger', 'cheetah', 'brown bear', 'American black bear', 'polar bear', 'sloth bear', 'mongoose', 'meerkat', 'tiger beetle', 'ground beetle', 'longhorn beetle', 'leaf beetle', 'dung beetle', 'rhinoceros beetle', 'weevil', 'bee', 'ant', 'grasshopper', 'cricket', 'cockroach', 'lacewing', 'ringlet', 'gossamer-winged butterfly', 'sea cucumber', 'cottontail rabbit', 'hare', 'Angora rabbit', 'porcupine', 'fox squirrel', 'marmot', 'beaver', 'guinea pig', 'common sorrel', 'zebra', 'pig', 'wild boar', 'warthog', 'hippopotamus', 'ox', 'water buffalo', 'bison', 'ram', 'bighorn sheep', 'Alpine ibex', 'hartebeest', 'impala', 'gazelle', 'dromedary', 'llama', 'weasel', 'mink', 'European polecat', 'black-footed ferret', 'otter', 'skunk', 'badger', 'armadillo', 'three-toed sloth', 'orangutan', 'gorilla', 'chimpanzee', 'gibbon', 'siamang', 'guenon', 'patas monkey', 'baboon', 'macaque', 'langur', 'black-and-white colobus', 'proboscis monkey', 'marmoset', 'white-headed capuchin', 'howler monkey', 'titi', "Geoffroy's spider monkey", 'common squirrel monkey', 'ring-tailed lemur', 'indri', 'Asian elephant', 'African bush elephant', 'red panda', 'giant panda', 'snoek', 'eel', 'coho salmon', 'rock beauty', 'sturgeon', 'garfish', 'pufferfish', 'abaya', 'academic gown', 'accordion', 'airliner', 'airship', 'ambulance', 'amphibious vehicle', 'analog clock', 'waste container', 'assault rifle', 'backpack', 'balance beam', 'balloon', 'ballpoint pen', 'Band-Aid', 'banjo', 'baluster', 'barber chair', 'barbershop', 'barn', 'barometer', 'barrel', 'wheelbarrow', 'baseball', 'bassinet', 'bassoon', 'bathtub', 'station wagon', 'lighthouse', 'beaker', 'military cap', 'beer bottle', 'bell-cot', 'tandem bicycle', 'ring binder', 'binoculars', 'boathouse', 'bobsleigh', 'bolo tie', 'poke bonnet', 'bookcase', 'bookstore', 'bottle cap', 'bow', 'brass', 'bra', 'breakwater', 'breastplate', 'broom', 'bucket', 'buckle', 'bulletproof vest', 'high-speed train', 'butcher shop', 'taxicab', 'cauldron', 'candle', 'cannon', 'can opener', 'tool kit', 'carton', 'car wheel', 'automated teller machine', 'cassette', 'cassette player', 'castle', 'catamaran', 'cello', 'chain', 'chain-link fence', 'chain mail', 'chainsaw', 'chest', 'chiffonier', 'chime', 'Christmas stocking', 'church', 'movie theater', 'cleaver', 'cliff dwelling', 'cloak', 'clogs', 'cocktail shaker', 'coffeemaker', 'coil', 'combination lock', 'computer keyboard', 'convertible', 'corkscrew', 'cornet', 'cowboy boot', 'cowboy hat', 'cradle', 'crane (machine)', 'crash helmet', 'crate', 'Crock Pot', 'croquet ball', 'cuirass', 'dam', 'desktop computer', 'diaper', 'digital clock', 'digital watch', 'dining table', 'dishwasher', 'disc brake', 'dock', 'dog sled', 'dome', 'doormat', 'drilling rig', 'drum', 'drumstick', 'dumbbell', 'Dutch oven', 'electric fan', 'electric guitar', 'electric locomotive', 'entertainment center', 'envelope', 'espresso machine', 'face powder', 'feather boa', 'filing cabinet', 'fire screen sheet', 'flagpole', 'flute', 'folding chair', 'forklift', 'fountain', 'fountain pen', 'four-poster bed', 'freight car', 'French horn', 'frying pan', 'fur coat', 'garbage truck', 'gas mask', 'golf ball', 'golf cart', 'gondola', 'gong', 'greenhouse', 'grille', 'grocery store', 'guillotine', 'barrette', 'half-track', 'hammer', 'hamper', 'hair dryer', 'hand-held computer', 'handkerchief', 'harmonica', 'harp', 'harvester', 'hatchet', 'home theater', 'honeycomb', 'hook', 'hoop skirt', 'horizontal bar', 'horse-drawn vehicle', 'hourglass', 'clothes iron', "jack-o'-lantern", 'jeep', 'T-shirt', 'jigsaw puzzle', 'pulled rickshaw', 'joystick', 'knee pad', 'lab coat', 'ladle', 'lampshade', 'laptop computer', 'lawn mower', 'paper knife', 'library', 'lighter', 'limousine', 'lipstick', 'slip-on shoe', 'lotion', 'speaker', 'loupe', 'sawmill', 'magnetic compass', 'mail bag', 'tank suit', 'manhole cover', 'maraca', 'marimba', 'mask', 'match', 'maypole', 'maze', 'measuring cup', 'megalith', 'microphone', 'microwave oven', 'milk can', 'minibus', 'minivan', 'missile', 'mitten', 'mobile home', 'Model T', 'modem', 'monastery', 'moped', 'mortar', 'square academic cap', 'mosque', 'mosquito net', 'scooter', 'mountain bike', 'tent', 'computer mouse', 'mousetrap', 'moving van', 'muzzle', 'nail', 'neck brace', 'nipple', 'notebook computer', 'obelisk', 'oboe', 'ocarina', 'oil filter', 'organ', 'oscilloscope', 'overskirt', 'bullock cart', 'oxygen mask', 'packet', 'paddle', 'paddle wheel', 'padlock', 'paintbrush', 'palace', 'pan flute', 'paper towel', 'parachute', 'parallel bars', 'park bench', 'passenger car', 'patio', 'payphone', 'pedestal', 'pencil sharpener', 'Petri dish', 'photocopier', 'plectrum', 'Pickelhaube', 'pickup truck', 'pier', 'piggy bank', 'pill bottle', 'ping-pong ball', 'pirate ship', 'pitcher', 'hand plane', 'planetarium', 'plastic bag', 'plate rack', 'plow', 'plunger', 'Polaroid camera', 'pole', 'police van', 'billiard table', 'pot', "potter's wheel", 'power drill', 'prayer rug', 'printer', 'prison', 'projectile', 'projector', 'hockey puck', 'punching bag', 'quill', 'quilt', 'race car', 'radiator', 'radio', 'radio telescope', 'rain barrel', 'recreational vehicle', 'reel', 'reflex camera', 'remote control', 'restaurant', 'revolver', 'rifle', 'rocking chair', 'rotisserie', 'eraser', 'ruler', 'running shoe', 'safety pin', 'sandal', 'sarong', 'saxophone', 'scabbard', 'weighing scale', 'school bus', 'scoreboard', 'CRT screen', 'screw', 'screwdriver', 'shield', 'shoji', 'shopping basket', 'shopping cart', 'shovel', 'shower cap', 'ski', 'ski mask', 'sleeping bag', 'slide rule', 'sliding door', 'slot machine', 'snorkel', 'snowmobile', 'snowplow', 'soap dispenser', 'soccer ball', 'sock', 'solar thermal collector', 'sombrero', 'space bar', 'space heater', 'space shuttle', 'spatula', 'motorboat', 'sports car', 'spotlight', 'steam locomotive', 'through arch bridge', 'steel drum', 'stethoscope', 'stone wall', 'stopwatch', 'stove', 'tram', 'stretcher', 'couch', 'stupa', 'submarine', 'suit', 'sundial', 'sunglass', 'sunscreen', 'suspension bridge', 'sweatshirt', 'swimsuit', 'swing', 'switch', 'table lamp', 'tank', 'tape player', 'television', 'tennis ball', 'thatched roof', 'front curtain', 'thimble', 'threshing machine', 'throne', 'tile roof', 'toaster', 'tobacco shop', 'toilet seat', 'torch', 'totem pole', 'tow truck', 'toy store', 'tractor', 'semi-trailer truck', 'trench coat', 'tricycle', 'trimaran', 'tripod', 'triumphal arch', 'trolleybus', 'tub', 'typewriter keyboard', 'umbrella', 'unicycle', 'upright piano', 'vacuum cleaner', 'vault', 'velvet', 'viaduct', 'violin', 'waffle iron', 'wall clock', 'wallet', 'wardrobe', 'military aircraft', 'sink', 'water jug', 'whiskey jug', 'whistle', 'window screen', 'window shade', 'Windsor tie', 'wok', 'wooden spoon', 'wool', 'split-rail fence', 'shipwreck', 'yawl', 'yurt', 'website', 'comic book', 'crossword', 'traffic sign', 'dust jacket', 'plate', 'consomme', 'hot pot', 'trifle', 'baguette', 'pretzel', 'cabbage', 'broccoli', 'cauliflower', 'zucchini', 'spaghetti squash', 'acorn squash', 'butternut squash', 'artichoke', 'bell pepper', 'cardoon', 'mushroom', 'strawberry', 'fig', 'jackfruit', 'custard apple', 'hay', 'carbonara', 'chocolate syrup', 'meatloaf', 'pot pie', 'eggnog', 'alp', 'cliff', 'coral reef', 'geyser', 'lakeshore', 'promontory', 'shoal', 'seashore', 'valley', 'volcano', 'baseball player', 'scuba diver', 'daisy', "yellow lady's slipper", 'corn', 'acorn', 'rose hip', 'horse chestnut seed', 'coral fungus', 'gyromitra', 'stinkhorn mushroom', 'earth star', 'hen-of-the-woods', 'bolete', 'ear']

The prediction set is: ['electric ray', 'stingray']

The prediction set is: ['loggerhead sea turtle', 'leatherback sea turtle', 'terrapin', 'box turtle']