数据级方法 (Data-Level Adaptation, DLA)：如果基础数据集不足以支持目标任务的训练，DLA通过引入更多数据来解决问题。例如，在分类任务中，利用基础数据集中的其他类别图像进行预训练。

参数级方法 (Parameter-Level Adaptation, PLA)：从参数空间角度优化模型，通过限制模型的自由度、使用正则化等方法防止过拟合。这种方法依赖于元学习技术，通过在大型参数空间中搜索最佳模型。

基于梯度的元学习 (Gradient-Based Meta-Learning, GBML)

匹配网络 (Matching Networks)

原型网络 (Prototypical Networks)

关系网络 (Relation Networks)

引入依赖包

pip install ftfy regex tqdmpip install git+https://github.com/openai/CLIP.gitimport numpy as npfrom packaging import pkg_resourcesprint("Torch version:", torch.__version__)

加载模型

import clipclip.available_modelsmodel, preprocess = clip.load("ViT-B/32")model = model.cuda().eval()input_resolution = model.visual.input_resolutioncontext_length = model.context_lengthvocab_size = model.vocab_sizeprint(f"Model parameters: {sum(int(np.prod(p.shape)) for p in model.parameters())}:")print("Input resolution:", input_resolution)print("Context length:", context_length)print("Vocab size:", vocab_size)

图像预处理与特征提取

import osimport skimagefrom PIL import Imageimport numpy as npimport torchfrom collections import OrderedDictfrom packaging import pkg_resourcesplt.figure(figsize=(16, 5))descriptions = {    "page": "a page of text about segmentation",    "chelsea": "a facial photo of a tabby cat",    "astronaut": "a portrait of an astronaut with the American flag",    "rocket": "a rocket standing on a launchpad",    "motorcycle_right": "a red motorcycle standing in a garage",    "camera": "a person looking at a camera on a tripod",    "horse": "a black-and-white silhouette of a horse",    "coffee": "a cup of coffee on a saucer"}original_images = []texts = []for filename in os.listdir(skimage.data_dir) if filename.endswith(".png") or filename.endswith(".jpg"):    name = os.path.splitext(filename)[0]    if name not in descriptions:        continue    image = Image.open(os.path.join(skimage.data_dir, filename)).convert("RGB")    plt.subplot(2, 4, len(images) + 1)    plt.imshow(image)    plt.title(f"{filename}\n{descriptions[name]}")    plt.xticks([])    plt.yticks([])    original_images.append(image)    texts.append(descriptions[name])

特征归一化与相似性计算

image_input = torch.tensor(np.stack(images)).cuda()text_tokens = clip.tokenize(["This is " + desc for desc in texts]).cuda()with torch.no_grad():    image_features = model.encode_image(image_input).float()    text_features = model.encode_text(text_tokens).float()image_features /= image_features.norm(dim=-1, keepdim=True)text_features /= text_features.norm(dim=-1, keepdim=True)similarity = text_features.cpu().numpy().Tcount = len(descriptions)plt.figure(figsize=(20, 14))plt.imshow(similarity, vmin=0.1, vmax=0.3)plt.colorbar()plt.yticks(range(count), [texts[i] for i in range(count)], fontsize=18)plt.xticks([])for i, image in enumerate(original_images):    plt.imshow(image, extent=(i - 0.5, i + 0.5, -1.6, -0.6), origin="lower")    for x in range(similarity.shape[1]):        for y in range(similarity.shape[0]):            plt.text(x, y, f"{similarity[y, x]:.2f}", ha="center", va="center", size=12)    for side in ["left", "top", "right", "bottom"]:        plt.gca().spines[side].set_visible(False)    plt.xlim([-0.5, count - 0.5])    plt.ylim([count + 0.5, -2])    plt.title("Cosine similarity between text and image features", size=20)

零样本分类示例

from torchvision.datasets import CIFAR100cifar100 = CIFAR100(os.path.expanduser("~/.cache"), transform=preprocess, download=True)text_descriptions = [f"This is a photo of a {label}" for label in cifar100.classes]text_tokens = clip.tokenize(text_descriptions).cuda()with torch.no_grad():    text_features = model.encode_text(text_tokens).float()    text_features /= text_features.norm(dim=-1, keepdim=True)    text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)    top_probs, top_labels = text_probs.cpu().topk(5, dim=-1)plt.figure(figsize=(16, 16))for i, image in enumerate(original_images):    plt.subplot(4, 4, 2 * i + 1)    plt.imshow(image)    plt.axis("off")    plt.subplot(4, 4, 2 * i + 2)    y = np.arange(top_probs.shape[-1])    plt.grid()    plt.barh(y, top_probs[i])    plt.gca().invert_yaxis()    plt.gca().set_axisbelow(True)    plt.yticks(y, [cifar100.classes[index] for index in top_labels[i].numpy()])    plt.xlabel("probability")    plt.subplots_adjust(wspace=0.5)plt.show()