Recent few-shot methods, such as parameter-efficient fine-tuning (PEFT) and
pattern exploiting training (PET), have achieved impressive results in
label-scarce settings. However, they are difficult to employ since they are
subject to high variability from manually crafted prompts, and typically
require billion-parameter language models to achieve high accuracy. To address
these shortcomings, we propose SetFit (Sentence Transformer Fine-tuning), an
efficient and prompt-free framework for few-shot fine-tuning of Sentence
Transformers (ST). SetFit works by first fine-tuning a pretrained ST on a small
number of text pairs, in a contrastive Siamese manner. The resulting model is
then used to generate rich text embeddings, which are used to train a
classification head. This simple framework requires no prompts or verbalizers,
and achieves high accuracy with orders of magnitude less parameters than
existing techniques. Our experiments show that SetFit obtains comparable
results with PEFT and PET techniques, while being an order of magnitude faster
to train. We also show that SetFit can be applied in multilingual settings by
simply switching the ST body. Our code is available at
https://github.com/huggingface/setfit and our datasets at
https://huggingface.co/setfit .
%0 Generic
%1 tunstall2022efficient
%A Tunstall, Lewis
%A Reimers, Nils
%A Jo, Unso Eun Seo
%A Bates, Luke
%A Korat, Daniel
%A Wasserblat, Moshe
%A Pereg, Oren
%D 2022
%K fewshot learning
%T Efficient Few-Shot Learning Without Prompts
%U http://arxiv.org/abs/2209.11055
%X Recent few-shot methods, such as parameter-efficient fine-tuning (PEFT) and
pattern exploiting training (PET), have achieved impressive results in
label-scarce settings. However, they are difficult to employ since they are
subject to high variability from manually crafted prompts, and typically
require billion-parameter language models to achieve high accuracy. To address
these shortcomings, we propose SetFit (Sentence Transformer Fine-tuning), an
efficient and prompt-free framework for few-shot fine-tuning of Sentence
Transformers (ST). SetFit works by first fine-tuning a pretrained ST on a small
number of text pairs, in a contrastive Siamese manner. The resulting model is
then used to generate rich text embeddings, which are used to train a
classification head. This simple framework requires no prompts or verbalizers,
and achieves high accuracy with orders of magnitude less parameters than
existing techniques. Our experiments show that SetFit obtains comparable
results with PEFT and PET techniques, while being an order of magnitude faster
to train. We also show that SetFit can be applied in multilingual settings by
simply switching the ST body. Our code is available at
https://github.com/huggingface/setfit and our datasets at
https://huggingface.co/setfit .
@misc{tunstall2022efficient,
abstract = {Recent few-shot methods, such as parameter-efficient fine-tuning (PEFT) and
pattern exploiting training (PET), have achieved impressive results in
label-scarce settings. However, they are difficult to employ since they are
subject to high variability from manually crafted prompts, and typically
require billion-parameter language models to achieve high accuracy. To address
these shortcomings, we propose SetFit (Sentence Transformer Fine-tuning), an
efficient and prompt-free framework for few-shot fine-tuning of Sentence
Transformers (ST). SetFit works by first fine-tuning a pretrained ST on a small
number of text pairs, in a contrastive Siamese manner. The resulting model is
then used to generate rich text embeddings, which are used to train a
classification head. This simple framework requires no prompts or verbalizers,
and achieves high accuracy with orders of magnitude less parameters than
existing techniques. Our experiments show that SetFit obtains comparable
results with PEFT and PET techniques, while being an order of magnitude faster
to train. We also show that SetFit can be applied in multilingual settings by
simply switching the ST body. Our code is available at
https://github.com/huggingface/setfit and our datasets at
https://huggingface.co/setfit .},
added-at = {2023-03-01T11:32:17.000+0100},
author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren},
biburl = {https://www.bibsonomy.org/bibtex/2c26639cfb14efd1e7fdabe7fc51c2223/daschloer},
description = {Efficient Few-Shot Learning Without Prompts},
interhash = {fdf4bf6757bb64b16f1a0d0acb1e36c1},
intrahash = {c26639cfb14efd1e7fdabe7fc51c2223},
keywords = {fewshot learning},
note = {cite arxiv:2209.11055},
timestamp = {2023-03-01T11:32:50.000+0100},
title = {Efficient Few-Shot Learning Without Prompts},
url = {http://arxiv.org/abs/2209.11055},
year = 2022
}