Contrastive models like CLIP have been shown to learn robust representations
of images that capture both semantics and style. To leverage these
representations for image generation, we propose a two-stage model: a prior
that generates a CLIP image embedding given a text caption, and a decoder that
generates an image conditioned on the image embedding. We show that explicitly
generating image representations improves image diversity with minimal loss in
photorealism and caption similarity. Our decoders conditioned on image
representations can also produce variations of an image that preserve both its
semantics and style, while varying the non-essential details absent from the
image representation. Moreover, the joint embedding space of CLIP enables
language-guided image manipulations in a zero-shot fashion. We use diffusion
models for the decoder and experiment with both autoregressive and diffusion
models for the prior, finding that the latter are computationally more
efficient and produce higher-quality samples.
Description
Hierarchical Text-Conditional Image Generation with CLIP Latents
%0 Generic
%1 ramesh2022hierarchical
%A Ramesh, Aditya
%A Dhariwal, Prafulla
%A Nichol, Alex
%A Chu, Casey
%A Chen, Mark
%D 2022
%K cs.CV
%T Hierarchical Text-Conditional Image Generation with CLIP Latents
%U http://arxiv.org/abs/2204.06125
%X Contrastive models like CLIP have been shown to learn robust representations
of images that capture both semantics and style. To leverage these
representations for image generation, we propose a two-stage model: a prior
that generates a CLIP image embedding given a text caption, and a decoder that
generates an image conditioned on the image embedding. We show that explicitly
generating image representations improves image diversity with minimal loss in
photorealism and caption similarity. Our decoders conditioned on image
representations can also produce variations of an image that preserve both its
semantics and style, while varying the non-essential details absent from the
image representation. Moreover, the joint embedding space of CLIP enables
language-guided image manipulations in a zero-shot fashion. We use diffusion
models for the decoder and experiment with both autoregressive and diffusion
models for the prior, finding that the latter are computationally more
efficient and produce higher-quality samples.
@misc{ramesh2022hierarchical,
abstract = {Contrastive models like CLIP have been shown to learn robust representations
of images that capture both semantics and style. To leverage these
representations for image generation, we propose a two-stage model: a prior
that generates a CLIP image embedding given a text caption, and a decoder that
generates an image conditioned on the image embedding. We show that explicitly
generating image representations improves image diversity with minimal loss in
photorealism and caption similarity. Our decoders conditioned on image
representations can also produce variations of an image that preserve both its
semantics and style, while varying the non-essential details absent from the
image representation. Moreover, the joint embedding space of CLIP enables
language-guided image manipulations in a zero-shot fashion. We use diffusion
models for the decoder and experiment with both autoregressive and diffusion
models for the prior, finding that the latter are computationally more
efficient and produce higher-quality samples.},
added-at = {2023-07-12T04:07:51.000+0200},
author = {Ramesh, Aditya and Dhariwal, Prafulla and Nichol, Alex and Chu, Casey and Chen, Mark},
biburl = {https://www.bibsonomy.org/bibtex/28bdd726dc6fc5fdda6ac18d480151562/aerover},
description = {Hierarchical Text-Conditional Image Generation with CLIP Latents},
interhash = {1bff5cc9a8d9f60a7d1572431e3c4f96},
intrahash = {8bdd726dc6fc5fdda6ac18d480151562},
keywords = {cs.CV},
note = {cite arxiv:2204.06125},
timestamp = {2023-07-12T04:09:24.000+0200},
title = {Hierarchical Text-Conditional Image Generation with CLIP Latents},
url = {http://arxiv.org/abs/2204.06125},
year = 2022
}