%%%%%%%%%%%%%%%%%%%%%%% file template.tex %%%%%%%%%%%%%%%%%%%%%%%%%
%
% This is a general template file for the LaTeX package SVJour3
% for Springer journals.          Springer Heidelberg 2010/09/16
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\RequirePackage{fix-cm}
%\documentclass{svjour3}                     % onecolumn (standard format)
%\documentclass[smallcondensed]{svjour3}     % onecolumn (ditto)
%\documentclass[smallextended]{svjour3}       % onecolumn (second format)
\documentclass[twocolumn]{svjour3}          % twocolumn
%
\smartqed  % flush right qed marks, e.g. at end of proof
%
\usepackage{graphicx}
\usepackage{times}
\usepackage{amsmath}
\usepackage{multirow}
\usepackage{color}
\usepackage{subfigure}
\usepackage{overpic}
\usepackage{ulem}
\usepackage[pagebackref=false,breaklinks=true,letterpaper=true,colorlinks,linkcolor=blue,citecolor=blue,urlcolor=blue]{hyperref}
\normalem


\newcommand{\figref}[1]{Fig.~\ref{#1}}
\newcommand{\tabref}[1]{Table~\ref{#1}}
\newcommand{\equref}[1]{Equ.~(\ref{#1})}
\newcommand{\secref}[1]{Section~\ref{#1}}
\newcommand{\cmm}[1]{\textcolor[rgb]{0,0.6,0}{#1}}
\newcommand{\todo}[1]{{\textcolor{red}{\bf [#1]}}}
\newcommand{\delete}[1]{{\textcolor[rgb]{0.8,0.8,0.8}{#1}}}
\newcommand{\vnudge}{\vspace{-.0in}}
\newcommand{\vBnudge}{\vspace{-.0in}}
\newcommand{\mypara}[1]{\noindent{\textbf{#1.}} \hspace{.05in}}
\newcommand{\salientcut}{SaliencyCut }
\newcommand{\mcols}[1]{\multicolumn{2}{c}{#1}}
\newcommand{\Cols}[1]{\multicolumn{2}{c|}{#1}}
\newcommand{\Rows}[1]{\multirow{2}{*}{#1}}
\newcommand{\colInf}{Top 50 & Top 100}
\newcommand{\ColInf}{\small{Flickr} & \small{Our} & \scriptsize{SHoG}}
\newcommand{\SHoG}{SHoG~\cite{11tvcg/eitz_sbir}}
\newcommand{\sArt}{state-of-the-art}
\newcommand{\etal}{et al.}
\graphicspath{{./Imgs/}}


\journalname{The Visual Computer} % Insert the name of "your journal" with

\begin{document}

\title{SalientShape: Group Saliency in Image Collections}

%\titlerunning{Short form of title}        % if too long for running head

\author{Ming-Ming Cheng \and Niloy J. Mitra \and Xiaolei Huang \and Shi-Min Hu }

%\authorrunning{Cheng et. al.} % if too long for running head

\institute{M.M. Cheng was in Tsinghua University when doing the major part
    of this work. He is now with Oxford Brookes University. \email{cmm.thu@gmail.com}.
    N.J. Mitra is with UCL.
    X. Huang is with Lehigh University.
    S.M. Hu is with TNList, Tsinghua University, Beijing.
}

\date{Received: date / Accepted: date}
% The correct dates will be entered by the editor


\maketitle

\begin{abstract}

Efficiently identifying salient objects in large image collections
is essential for many applications including image retrieval,
surveillance, image annotation, and object recognition.
%
We propose a simple, fast, and effective algorithm for locating and
segmenting salient objects by analysing image collections.
As a key novelty, we introduce {\em group saliency} to achieve superior unsupervised
salient object segmentation by extracting salient objects (in collections of pre-filtered images)
that maximize between-image similarities and within-image distinctness.
%
To evaluate our method,
we construct a large benchmark dataset consisting of 15K images across multiple categories with 6000+
pixel-accurate ground truth annotations for salient object regions where applicable.
%
In all our tests, group saliency consistently outperforms state-of-the-art
single-image saliency algorithms, resulting in both higher precision and better recall.
%
Our algorithm successfully handles image collections, of an order larger than any existing benchmark datasets,
consisting of diverse and heterogeneous images from various internet sources.


\keywords{Saliency detection \and group saliency \and object of interest segmentation \and image retrieval}
% \PACS{PACS code1 \and PACS code2 \and more}
% \subclass{MSC code1 \and MSC code2 \and more}
\end{abstract}


%-------------------------------------------------------------------------------
\section{Introduction}

The ubiquity of acquisition devices, e.g., cameras and smart-phones,
and the growing popularity of social media have resulted in an explosion of
digital images accessible in the form of personal and internet photo-collections.
Typically, such image collections are huge in size, have heterogeneous content,
and are noisy due to diverse background and illumination conditions.
Although such images form a well-established communication medium for
sharing experiences or blogging events,
we still lack efficient and effective methods to analyze and organize such images.


Determining characteristic or salient regions of images allows
transitioning from low-level pixels to more meaningful high-level regions,
and thus form an essential step for many computer graphics and computer vision applications,
including interactive image editing
~\cite{tog/ChengZMHH10,ChiaColorization,margolin2013saliency,cgf/ZhangC09,zcczhm_interactiveImages_sigg12},
image retrieval \cite{tvcg/Chen,gao20123,Gao2011,he2012mobile},
and internet visual media processing \cite{ChenCT09,hu2013internet,huang2011arcimboldo,11tvc/Liu}.
%
Recently, significant success has been reported in saliency-based image segmentation producing near
ground-truth performance on simple images
(\cite{09cvpr/Achanta_FTSaliency,ChengPAMI,11cvpr/Cheng,margolin2013saliency} and references therein).
%
The next challenge is to reliably segment salient object regions in large heterogeneous
image collections such as internet images, e.g., Flickr, Picasa.
Since such collections contain rich information about our surroundings,
their effective analysis will naturally provide improved understanding of image contents.
%, e.g., what are the salient objects in the images, or how do they vary in appearances.


\if0
\cmm{Among various visual attributes for analyzing images, such as color, texture, shape, etc.,
shape of salient object region is the most important \cite{99ACMMM_BouetShapeRetrieval}.
Humans can easily recognize thousands of objects entirely by their shapes
\cite{97Cognition_hoffman_salience}.
Finding salient object regions and their geometry relations has been shown to be an
effective way to enable large scale meaningful interactive image editing applications
\cite{cgf/ZhangC09,tog/ChengZMHH10,zcczhm_interactiveImages_sigg12}.
Such high level image processing applications are mostly limited to interactive editing
of a few images.
For many key problems of using vast amount existing images, e.g. content based image retrieval,
the less discriminative (e.g. color, texture, etc.) but easy available features are still the main stream \cite{blobWorld02,csur08/datta}.
A major difficulty is to fully automatically extracting {\em entire} salient object regions.
Recently,
}
%Traditionally,content-based image retrieval systems (e.g.,
%\mbox{\cite{97Cognition_hoffman_salience,99ACMMM_BouetShapeRetrieval,blobWorld02,csur08/datta}})
%analyze {\em single} images using one or multiple visual attributes
%like color, texture, shape, salient features, and salient image regions.
%
significant success has been reported in saliency-based image segmentation producing near
ground-truth performance on simple images~\cite{09cvpr/Achanta_FTSaliency,11cvpr/Cheng}.
%
The challenge, however, is to reliably segment salient object regions in large heterogeneous
image collections such as internet images, e.g., Flickr, Picasa.
Since such collections contain rich information about our surrounding world
~\cite{11tog/ChiaColorization,11tvc/Liu},
their effective organization will naturally provide improved understanding of image contents.
%, e.g., what salient objects are in the images, or how are their appearances.
\fi

\begin{figure*}[ht!]
    \centering
    \includegraphics[width=\textwidth]{Pipeline}\\
    \caption{System pipeline. Our system explicitly extracts salient object regions from
        a group of related images with heterogeneous quality offline~(a-d, f) to enable
        efficient online~(e) shape based query.
        To enable effective salient object segmentation for a large
        collection of images with heterogenous contents,
        our system only requires a simple input text keyword and a coarse sketch (a),
        for initial query of internet image candidates (b) and shape ranking (e).
        For a new query shape related to a processed keyword,
        the segmentation results (d) can be re-used for efficient query
        (only shape matching is required in this typical use case).
    }\label{fig:pipleLine}
\end{figure*}



We introduce {\em SalientShape},
a group-saliency based framework for salient object detection and segmentation in image collections.
%
We demonstrate that even when the shared content across image collections is small, e.g., 30\%,
our framework produces superior results as compared to individually processing the images.
%
Our proposed method is simple, scales well with increasing size of collections,
 has low memory-footprint, and can be effectively integrated with existing image handling systems.


Our algorithm (see \figref{fig:pipleLine}) runs in the following key stages:
%
First, for a query object class, we retrieve candidate images by pre-filtering using keywords.
Such retrieved images are usually noisy and contain outliers
due to limitations in keyword-based image search, ambiguity of keywords,
and heterogeneous tags.
%
Next, we detect and segment salient object regions of each candidate image using
SaliencyCut~\cite{11cvpr/Cheng} and automatically remove candidates with fragmented scenes
or with unreliable segmentation quality by jointly analyzing the salient regions.
%
Then, in a key stage, we re-rank the remaining candidates based on the consistency between
their saliency cuts and the user provided sketches.
In this step, access to image collections proves critical since even in noisy ensembles
we observe that  segments corresponding to the inlier objects have {\em consistent} appearance and shape properties.
To exploit this, we build global (group) foreground and background appearance models
from the top-ranked candidate images for the query object class.
Finally, we use the extracted appearance model for group saliency region detection and segmentation.
%
We iterate the process to alternately improve saliency estimates and
appearance models. %, and in turn the segmentation and retrieval.



We compare the resultant segmentation results with state-of-the-art single
image salient region segmentation methods
~\cite{09cvpr/Achanta_FTSaliency,10eccv/rahtu_segmenting,11cvpr/Cheng} (see \figref{fig:plotsCut}),
and to retrieval performance with \SHoG$ $ on 30 categories (see \tabref{tab:statistic}).
%
Further, we introduce a benchmark dataset consisting of 15,000 images collected from
\href{http://www.flickr.com/}{Flickr} along with 6000+ pixel-accurate  ground truth salient
object masks where applicable (to be made publicly available for academic use).
 To the best of our knowledge,
our benchmark dataset with pixel accurate salient object region ground truth labeling
is the largest of its kind ($15 \times$ larger than \cite{09cvpr/Achanta_FTSaliency}),
while the images are more difficult and closer to real-world scenarios.
In our extensive tests, group saliency consistently outperforms existing state-of-the-art alternatives,
especially on images with cluttered backgrounds.


The improved performance is primarily due to the joint saliency
estimation and (single-image and group/global) appearance models learning.
%
Our system also benefits from meta-data\footnote{Meta-data is the current industry standard
for image retrieval as popularized by search engines like
\href{http://images.google.com/}{Google image}, \href{http://www.flickr.com/}{Flickr}, etc.},
visual saliency, and shape similarity to explicitly detect salient object regions
and enable shape retrieval without influence from background clutter.
%Further, improved SaliencyCut allows us to naturally make use of existing shape matching algorithms
%to achieve scale-, rotation-, and translation-invariant retrieval
%~\cite{99ACMMM_BouetShapeRetrieval,01cvpr_MoriBM_SCRetrieval,04PR_ZhangL_Review}.
%
In summary, we (i)~introduce {\em group saliency} to  extract object
of interest regions from a group of correlated but heterogeneous images,
%improving state-of-the-art SBIR results,
and
% \item propose a set of simple measures for estimating the quality of unsupervised image segmentation,
%    %and a simple, efficient, and effective SBIR algorithm, and
% \item present a benchmark dataset for evaluating the performance of unsupervised object segmentation
%     from a group of (related) images and subsequent object shape based image retrieval.
(ii)~present a large benchmark dataset to objectively compare the superiority of group
saliency over traditional single image saliency detection.
%\end{itemize}
%
Since our focus is on {\em consistent segmentation}, we show retrieval only as a potential application rather than being the focus of this work.

%-------------------------------------------------------------------------------
\section{Related Work}


\mypara{Salient region extraction}
Various methods have been proposed for extracting salient regions from single images:
Ko and Nam~\cite{06josa/KoN_InterestSegmentation} select salient regions using
a support vector machine trained on image segment features,
and then cluster these regions to extract salient objects.
Han \etal~\cite{06TCSVT/han_unsupervised} model color, texture, and edge features
using a Markov random field framework and grow salient object regions from seed values
in the saliency maps.
Achanta \etal~\cite{09cvpr/Achanta_FTSaliency} average saliency values within
image segments produced by mean-shift segmentation, and find salient objects
by identifying image segments with average saliency above a threshold.
%
Cheng \etal~\cite{ChengPAMI,11cvpr/Cheng} propose a saliency estimation method to automatically
initialize an iterative version of GrabCut~\cite{04tog/rother_grabcut} to segment salient object regions.
%
These methods aim at salient region extraction for {\em individual} images, while
ignoring useful global information available from correlated image collections.
Recently, co-saliency methods  have been proposed to find common salient object(s)
between pair of images~\cite{10icip/chen_co,11tip/li_co} or among multiple
images~\cite{11cvpr/chang_co,zhang2012hierarchical}.
Such methods, however, require salient areas to contain parts of the foreground objects across most images.
Further, the algorithms are difficult to scale to large number of images
(largest demonstrated collection has 30 images).  In contrast, we focus on detecting and segmenting
correlated salient object regions from large (thousands or more)
image collections with heterogeneous contents (e.g., internet images).


\mypara{Internet image re-ranking}
%
Fergus et al.~\cite{04eccv/fergus_visual} use the top results returned
from a web-based image search engine to train a classifier,
and then use the classifier to filter the search results.
Ben-Haim \etal~\cite{06CVPRW_Haim_ImproveByCluster} automatically segment images
into regions and build color histogram features for each region,
which are then clustered to obtain principal modes.
The remaining images are then re-sorted based on the distance of their
regions to the mean feature values of the principal clusters.
Cui \etal~\cite{08ACMMM/cui_real} categorize a query image into one of several
predefined categories,
and employ a specific similarity measure in each category to combine image
features for re-ranking based on the query image.
Popescu \etal~\cite{09ACMMM/popescu_lightweight} re-rank images based on the visual coherence
of queries using a diversification function to avoid near-duplicate images and ensure
that different aspects of a query are presented to the user.
None of these algorithms use visual attention and shape information of the
desired object.
In contrast, we use such information to capture potential
appearances that a desired object class may have,
enabling superior salient region extraction.
%meanwhile supporting more intuitive SBIR, and knowing what's the exact range of object of interest.



\mypara{Sketch based image retrieval (SBIR)}
%
Early works by Hirata and Kato~\cite{hirata1992query} perform retrieval
by comparing shape similarity between user sketches and edge images in a database,
expecting precise sketches from the users.
Alberto and Pala~\cite{pami97/Bimbo} further employ elastic matching to a user-sketched
template for robust retrieval, with the cost of expensive computation.
Recently, Cao et al. developed a novel indexing technique~\cite{11cvpr/Cao} to support efficient
contour-based matching for a retrieval system~\cite{cao2010mindfinder} that handles millions of images.
However, the method does not provide translation, scale, or rotation invariance, and more importantly expects the desired
object to appear at roughly similar positions, scale, and rotation as in the user-drawn query sketch.

In an important recent system, Eitz \etal~\cite{11tvcg/eitz_sbir} use local descriptors to achieve
state-of-the-art retrieval performance.
Their success is mainly attributed to translation invariance
of local descriptors as well as using large local features
($20\%-25\%$ image's diagonal length) to retain large-scale image characteristics.
%However, for such large window size, there is not much space left for translation invariant.
All such methods compare user sketches with image edges (or boundaries),
suffering from influence of background edges when finding a desired object.
%
Salient object region extraction~\cite{ChenCT09,huang2011arcimboldo,tvcg/Chen}
and multi-resolution region representation~\cite{hu2011bag} have been used to handle background clutter.
We also use explicit region information to support SBIR.
However, instead of feature designing, matching, or indexing,
we use visual attention and (learned) global appearance information
to improve salient region extraction,
which naturally supports shape retrieval with scale, rotation, and translation variations.


\mypara{Segmentation transfer}
Our work is also related to recent advances in segmentation transfer.
Kuettel and Ferrari \cite{kuettel2012figure} transfers segmentation
mask from training windows that are visually similar to the target image windows.
In an impressive concurrent effect, Kuettel \etal~\cite{kuettel2012segmentation}
successfully generate pixelwise segmentations for ImageNet \cite{deng2009imagenet},
which contains 577 classes over 500K images,
by leveraging existing manual annotations in form of class label, bounding-boxes,
and external pixel-wise ground truth segmentations in the PASCAL VOC10 dataset~\cite{everingham2010pascal}.
These methods also use class-wise appearance models, captured by Gaussian Mixture Model,
and model the segmentation problem in an extended MRF framework.
%
However, in absence of appropriate methods to choose good segmentations before
segmentation propagation, the accuracy degrades gracefully over the stages.
Instead, we carefully choose good segmentations by measuring scene complexity,
imprecise cut, region incompleteness, and shape consistence.
This allow us to select reliable candidates leading to high quality global appearance, which accords with human
understanding about the classes (see also \figref{fig:Statistics} and supplemental materials
\footnote{\href{http://mmcheng.net/gsal/}{http://mmcheng.net/gsal/}}).
Thus, instead of external pixel-accurate ground truth labeling,
our method only requires a few (typically one is enough) sketches for each class
to help learn useful global appearance information,
thus significantly lowering required annotation efforts.



%-------------------------------------------------------------------------------
\section{Unsupervised Segmentation of Individual Candidate Images}
\label{sec:singleSal}

For any given keyword (e.g., dogs, jumping dogs, etc.),
we first retrieve a set of candidate images using Flickr,
typically around 3,000 (see \figref{fig:pipleLine}).
For each such image, we perform unsupervised segmentation to estimate a salient object,
as described next. The key stage comes later (\secref{sec:groupSaliency}) when
we exploit correlation in salient objects' appearance and shape among related images
for a query object class, towards group saliency.
%between similar images using group saliency,
%while ensuring consistency with input user-sketch.


%-------------------------------------------------------------------------------
\subsection{Saliency guided image segmentation}

We briefly describe our previous SaliencyCut \cite{11cvpr/Cheng} work,
which is used here for single image saliency estimation.
Segmenting a color image $I:=\{I_i\}$ consisting of pixels $I_i$ in the RGB color
space amounts to solving for corresponding \emph{opacity} values
$\alpha := \{\alpha_i\}$ with $\alpha_i \in \{0,1\}$ at each pixel.
%Note that for hard segmentation $\alpha_i \in \{0, 1\}$ with $0$ for background and $1$ for foreground.
%
We enable unsupervised segmentation by building a Gaussian Mixture Model~(GMM)
for foreground/background color distribution $G$,
which we then use to directly extract a binary segmentation mask to avoid manual thresholding.


We use GrabCut formulation~\cite{04tog/rother_grabcut} to model
single image saliency-based segmentation problem and use their suggested parameters.
The segmentation problem can be solved by optimizing a \emph{Gibbs} energy function $E$ as:
\begin{equation}\label{equ:GibbsEnergy}
   \min_{\alpha} E(\alpha, G, I) = \min_{\alpha} (U(\alpha, G, I) + V(\alpha, I))
\end{equation}
where, $U(\alpha, G, I)$ evaluates the fitness of the opacity distribution $\alpha$ with respect to
the data $I$ under a given color model $G$
and $V(\alpha, I)$ measures the smoothness of $\alpha$.
% (see also GrabCut formulation~\cite{04tog/rother_grabcut}).
%We iteratively minimize this energy to automatically extract a salient segmentation.
The fitness term $U(\alpha, G, I)$ of a pixel $I_i$ is defined as the negative
log probability of this pixel belonging to its corresponding color distribution $G$
($\alpha$ is a binary value. $\alpha = 1$ indicates a foreground GMM, while $\alpha = 0$ denotes a background GMM).
The smoothness term $V(\alpha, I)$ is defined as the sum of neighboring pixel color
similarity when they take different $\alpha$ values
(see \cite{04tog/rother_grabcut} for details about the measurement and its parameters estimation).
In SaliencyCut \cite{11cvpr/Cheng}, the continuous saliency values are thesholded
to automatically initialize foreground and background color models in GrabCut \cite{04tog/rother_grabcut}.
To improve  robustness to noisy automatic initialization, the segmentation process is iterated,
with morphological operations to improve performance (see \cite{11cvpr/Cheng} for more details).




\mypara{Implementation details} Unlike \cite{11cvpr/Cheng},
we use a new initialization procedure that avoids the unintuitive
threshold choosing process.
Foreground and  background are modeled with  color GMMs.
Instead of assigning pixel colors to a model using a threshold,
we treat every pixel color as a weighted sample that contributes to both
foreground and background color GMMs.
E.g. for a pixel with saliency value 0.7, we use weight factors 0.7 and 0.3
when building the foreground and background color GMMs respectively.
Although this soft assignment incurs a small computational overhead ($<10\%$)
in each SaliencyCut iteration,
it introduces more accurate initialization which  reduces the number of iterations required.
In the experiments, the overall segmentation quality and computational time
is similar to \cite{11cvpr/Cheng} with a manually chosen threshold.
During GMM estimation, we use the color quantization bin (see \cite{11cvpr/Cheng})
as a unit of samples instead of each pixel color for computational efficiency.

%In contrast to \cite{11cvpr/Cheng}, we use a continuous saliency map
%to learn an initial color GMM $G$.
%%
%Then we alternately extract binary opacity values and refine GMM estimates by
%optimizing Equation~\ref{equ:GibbsEnergy} using iterated graph-cuts,
%thus obtaining a binary cut of salient regions, i.e., SaliencyCut.
%%
%\cmm{Thus, we avoid having to choose a threshold to
%binarize the saliency map as in~\cite{11cvpr/Cheng}.
%%
%Empirically, such a soft assignment improves segmentation
%results with only a small computation overhead ($<10\%$).}



%-------------------------------------------------------------------------------
\subsection{Measuring the reliability of SaliencyCut}
%
Keyword based image retrieval often produces a large percentage of irrelevant images
(see also \secref{sec:EvalSeg} and supplementary materials).
%Unlike traditional retrieval applications, in our setting,
Luckily, in our retrieval application,
users are interested in the precision rate of the top ranked images (e.g., top 50 images)
rather than the recall rate of the entire searched results (typically a few 1000s).
Hence, we aggressively prune away likely outlier images, as described next.



\mypara{Scene complexity}
Saliency maps are often poor for complex/cluttered scenes.
We use the number of regions produced by segmentation~\cite{04ijcv/felzenszwalb}
as an indicator of scene complexity (see \figref{fig:ComplexScene}).
Intuitively, images with a small number of segments are simpler.
We sort the images based on increasing number of regions and retain only the top $T_{R}$ images for subsequent stages.
We use $T_{R}=70\%$ in our tests leading to around a thousand images being discarded.


Note that in our problem setting huge sets of candidates are available,
e.g., internet images, and the users simply want to easily find some high quality desired targets rather than explore the whole collection.
Hence, as a design choice, we decided to favor higher precision over higher recall. We empirically threshold values and use them for all our tests. This resulted in reliable global statistics for group segmentation.


\begin{figure}[t!]
    \centering
    \begin{overpic}[width=\linewidth]{cluttered.jpg}
        \put(6,0){\small original image}
        \put(37,0){\small segmentation \cite{04ijcv/felzenszwalb}}
        \put(75,0){\small saliency \cite{11cvpr/Cheng}}
    \end{overpic}
    \caption{Complex and cluttered scenes usually lead to segmentations composed of many small regions, or fragmented saliency maps.
    }\label{fig:ComplexScene}
\end{figure}


\begin{figure}[b!]
    \centering
    \begin{overpic}[width=\linewidth]{reliability.jpg}
        \put(6,0){(a)}
        \put(28,0){(b)}
        \put(53,0){(c)}
        \put(74,0){(d)}
        \put(90,0){(e)}
        \put(0,21){\scriptsize \textcolor{green}{$R_P = 93\%$}}
        \put(43,21){\scriptsize \textcolor{green}{$R_B = 88\%$}}
        \put(68,21){\scriptsize \textcolor{green}{$R_P = 12\%$}}
        \put(68,18){\scriptsize \textcolor{green}{$R_B = 51\%$}}
    \end{overpic}
    \caption{Different SaliencyCuts are marked in red in (a), (c), and (d).
        While (a) is imprecise and (c) produces an incomplete region-of-interest, (d) yields a good cut.
        Undesirable cuts are detected based
        on relevant foreground probability maps (b, e).
        Segmentation quality ranking scores
        (imprecise $R_P$ and incomplete $R_B$) are overlaid on top.
    }
    \label{fig:CutQuality}
\end{figure}

\mypara{Segmentation quality}
Even for relatively simple scenes, \salientcut can have imprecise or incomplete boundaries,
which we detect as follows:
%
(i)~For each image, we use its \salientcut region and its remaining parts
to train GMMs for foreground and background regions, respectively.
We then estimate the foreground probability of relevant image pixels according
to these two GMMs. Specifically,
we take the sum of foreground probabilities for pixels inside a narrow band
(of 30 pixel width) surrounding the \salientcut as a measure for imprecise cut
(e.g., \figref{fig:CutQuality}a):
the higher this sum, the lower the predicted quality of the cut.
(ii)~We take the total number of \salientcut region pixels within a narrow band
(of 20 pixel width) along the image border as a measure for incompleteness of
the object-of-interest region (see \figref{fig:CutQuality}c):
the higher this number, the more likely the cut object region is incomplete.
We sort the images according to increasing order of the above two measures
and retain the top $T_{P}$ and $T_{B}$ of images for subsequent stages.
We use $T_{P}=80\%$ and $T_{B}=80\%$ in our experiments.
The retained images are next analyzed for image collection consistency.


%-------------------------------------------------------------------------------
\section{Group Saliency}
\label{sec:groupSaliency}

Retrieved images in a collection (e.g., Flickr) are largely correlated,
but can have differences due to pose, appearance, etc.
(see \figref{fig:SBIR_demo} and also supplementary).
We use rough sketches as an indicator of the poses that the user is interested in
(e.g., when user searches for a specific style of `dog jump'),
while we use consistency across the retrieved images to extract
what are plausible appearance models for the salient object (e.g., color of the `dog').
%
Such saliency, which we refer to as \emph{group saliency}, favors both
\emph{similarities} between images and \emph{distinctness} within each image.
Specifically, we first use an efficient cascade model to rank
single image unsupervised segmentation results according to their shape consistency
with user input sketch.
Top ranked results are then used to train a global appearance statistical model for
refining group saliency and segmentation results.


\subsection{Cascade filtering for sketch based retrieval}
%
Having reliable \salientcut boundaries from initial images,
we benefit from existing shape matching algorithms
\cite{99ACMMM_BouetShapeRetrieval,04PR_ZhangL_Review} to retrieve desired images based
on consistency with user-input sketches.
We use the following simple measures which are easy to calculate on regions with
clean background:
\begin{itemize}
  \item circularity: $Perimeter^2/Area$ \cite{04PR_ZhangL_Review},
  \item solidity: $RegionArea/ConvexHullArea$~\cite{flickner1995query},
  \item Fourier Descriptor~\cite{02SPIC_GFD}, and
  \item Shape Context~\cite{02pami/BelongieMP}.
\end{itemize}


We proceed in a cascaded fashion.
For each measure, we sort the shapes in decreasing order based on their similarity to user
sketch, and retain a top percentage of the candidates.
%
The measures are arranged in increasing complexity, allowing efficient and early rejection
of candidates with large dissimilarity.
In our experiments, we keep $T_C = 80\%$,  $T_S = 80\%$, and $T_{F} = 70\%$ images
according to circularity, solidity, and Fourier descriptor respectively;
the corresponding dimensions for these descriptors are 1, 1, and 15.
We compare these descriptors using simple Euclidean distance with
corresponding features of the user sketch.
%
We finally use the Shape Context (with default parameters and matching methods
as suggested in the original paper \cite{02pami/BelongieMP}),
which is complex but effective to properly order the remaining candidates.
%
While one can employ more complex shape descriptors~(e.g.,~\cite{11tvcg/eitz_sbir}),
we find the above selection sufficiently diverse to prune out most shape outliers
(see also supplementary material).
%
Note that at this stage we are left with only
$T_R \cdot T_P \cdot T_B \cdot T_C \cdot T_S \cdot T_F \approx 20.0\%$ of the image shapes,
which are used for appearance consistency, as described next.


\begin{figure}[t!]
    \centering
    \begin{overpic}[width=\linewidth]{DogJumpP.jpg} \put(22,  0){$\cdots$} \end{overpic} \\
    \vspace{.1in}
    \begin{overpic}[width=\linewidth]{GiraffeP.jpg} \put(45.5,0){$\cdots$}\end{overpic}
    \caption{Appearance histogram of sample colors in object of
        interest region in `dog jump' images according to the learned global prior.
        %
        Left shows typical foreground colors, while right shows typical background colors.
        %
        Probability values $\{p\}$ are ordered and
        histogram height is $|p - 0.5|$.
        We ignore color samples with probability around 0.5 as chance.
        Inset shows a typical input image example.
    }
    \label{fig:Statistics}
\end{figure}


\subsection{Statistical global appearance models}\label{sec:GlobalStatis}
%
%
After cascade filtering, the top ranked images typically have high precision.
We use the top $50$ images as a high quality training set to learn a global appearance
prior to guide subsequent group saliency detection and segmentation.
We choose GMM models to capture such a prior $\bar{G}$ for two reasons:
(i)~GMM models generalize better on small amounts of training
data than histogram models~\cite{ijcv02/jones};
(ii)~GMM priors can be easily integrated in our unsupervised image segmentation
framework (see \secref{sec:singleSal}).
%
For example,  \figref{fig:Statistics} shows foreground and background
GMM models for the `dog jump' example indicating that dogs are typically yellow or dark in color and are
like to play on green grass/fields; for the 'giraffe' we find typical background colors consist of blue/green indicating sky/trees, as typically associated with context information for giraffes.
Although other attributes like texture and visual vocabulary can be considered,
we currently use only color. We empirically chose 8 Gaussians each to model major appearance of
foreground/background per category and found that this number is not sensitive.



Note that since shape features are typically  orthogonal to
appearance attributes, the samples we retrieved based on shape largely preserve their
appearance diversity and can be used to learn representative appearance models.
Typically only a fraction of such images ($15\%-57\%$ in our tests) contain
desired objects.
These objects may have different colors, textures, and even a single object
may comprise of several regions with very different appearance (e.g., butterfly).
%
We found that considering the largest appearance cluster~\cite{06CVPRW_Haim_ImproveByCluster} or
top-ranked internet images~\cite{04eccv/fergus_visual} as an initial set
to be unsatisfactory.
%In another related previous work,
In an interesting effort, Chang \etal~\cite{11cvpr/chang_co} use repeatedness among images
as a global prior of multiple images and  assume that most images contain at least parts of the foreground object,
an assumption that is often violated in our setting.
Further, since each image is compared with all others, the method cannot be used for large collections  (e.g., they considered image sets of maximum size $30$,
while we handle a few 1000s).


\begin{figure}[t!]
    \centering
    \begin{overpic}[width=\linewidth]{GroupSalImpr.jpg}
        \put(6,0){(a)}
        \put(22.8,0){(b)}
        \put(39.6,0){(c)}
        \put(56.4,0){(d)}
        \put(73.2,0){(e)}
        \put(90,0){(f)}
    \end{overpic}
    \caption{Examples of using statistics to refine unsupervised segmentation:
        (a)~source image, (b)~single image saliency map, (c)~SaliencyCut
        (using \equref{equ:GibbsEnergy}), (d)~global color prior,
        (e)~group saliency map, and (f)~group saliency segmentation (using \equref{equ:Global}).
        Note the improvement from (c) to (f) (see supplementary material).
    }
    \label{fig:GroupSal}
\end{figure}



\subsection{Estimating group saliency}
%
%
Finally, we use the learned global appearance statistics to improve the
saliency detection and \salientcut of each image.
Since the estimated global color prior $\bar{G}$ is encoded as GMMs,
we simply add a global prior constraint to our single image unsupervised segmentation
energy function of \equref{equ:GibbsEnergy}.
The new energy function takes the form:
\begin{equation}
   E(\alpha,G,\bar{G},I) = \lambda U(\alpha,\bar{G},I)+(1-\lambda)U(\alpha,G,I) +V(\alpha,I)
     \label{equ:Global}
\end{equation}
where, the additional term $U(\alpha, \bar{G}, I)$ evaluates the fitness of the opacity
distribution $\alpha$ to the global color prior $\bar{G}$,
while weight $\lambda$ ($0.3$ in our test) balances between global color prior
and per image color distribution.
Here the global color prior $\bar{G}$ reflects similarity between the targets,
while the per-image color distribution $G$ is trained according to the
individual image saliency map and captures distinctness within an image.


Similar to \equref{equ:GibbsEnergy}, we optimize \equref{equ:Global}
to get group saliency segmentation results.
We then encode group saliency maps as probability maps of pixels
belonging to the object-of-interest region obtained by group saliency segmentation.
%
Note that although the change compared to \equref{equ:GibbsEnergy} is small,
the improvement in estimated saliency is significant with only marginal computational overhead.
%The obtained object-of-interest region as well as other image regions
%are used to train local GMMs to model object-of-interest and background
%color distributions.
%The probability maps of each color belonging to object-of-interest region are taken
%as their group saliency maps.
%We locally average saliency values to uniformly highlight similar regions,
%and retain connected regions with high saliency values to suppress spurious background regions.


\figref{fig:GroupSal} demonstrates typical improvements in saliency cut
and segmentation using global color priors.
In the `dog jump' image, the green parts of the image are estimated to be more likely to
be background rather than foreground according to the learned
global color prior.
Similarly, in the plane example, missing object regions are correctly recovered with the help
of global statistics (see supplementary material for more examples).



\begin{figure}[b]
  \centering
  \includegraphics[width=\linewidth]{dataset.jpg}
  \caption{Example images from the benchmark dataset that correspond to the keyword `dog jump',
    with pixel-accurate  ground-truth labeling for the corresponding object of interest regions
    (if such a region exists; 4 out of the 7 here).
  } \label{fig:dataset}
\end{figure}



\begin{figure*}
  \centering
  \subfigure[Fixed thresholding saliency maps]{\includegraphics[width=.49\linewidth]{SalMap2}\label{fig:plots}}
  \hfill
  \subfigure[Object of interest region segmentation]{\begin{overpic}[width=.468\linewidth]{CutCmp}
       \small
       \put(6,  1){Butterfly}
       \put(24, 1){CoffeeMug}
       \put(44, 1){DogJump}
       \put(65, 1){Giraffe}
       \put(85, 1){Plane}
  \end{overpic} \label{fig:plotsCut}}
  \caption{Evaluation results on our benchmark dataset.
     (a) Precision-recall curves for naive thresholding of saliency maps.
     S, G1, G2 represent single image saliency, group saliency after the 1st and 2nd
     iterations, respectively.
     Subscripts B, C, D, G, P represent groups of `butterfly', `coffee mug',
     `dog jump', `giraffe', `plane', respectively.
    (b) Comparison of $F_{\beta}$ for image groups using single image saliency
    segmentation methods (FT~\cite{09cvpr/Achanta_FTSaliency},
    SEG~\cite{10eccv/rahtu_segmenting}, RC~\cite{11cvpr/Cheng}) vs.
    group saliency (GS) segmentation.
    The RC and GS in (b) corresponds to results of \equref{equ:GibbsEnergy} and
    \equref{equ:Global} respectively.
  }
\end{figure*}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiments}
%
%
We implemented our framework in C++ and evaluated it using a
Quad Core i7 920 CPUs with 6G RAM.
We use the group saliency based retrieval results to re-train new appearance
statistics and iteratively improve saliency segmentation (see \figref{fig:pipleLine}).
Experimentally, we found two rounds of iterations to be sufficient.


We evaluated the proposed method for three different applications using a benchmark dataset:
(i)~fixed thresholding of group saliency maps, (ii)~object of interest segmentation,
and (iii)~sketch based image retrieval.   For the first two applications,
we consider the average segmentation performance only over those images that do contain
the target object (according to annotated ground truth).





\subsection{Benchmark dataset for saliency segmentation}
We collected a labeled dataset of categorized images initially extracted by querying with
keywords from \href{http://www.flickr.com/}{Flickr}.
We downloaded about 3,000 images for each of the 5 keywords:
`butterfly', `coffee mug', `dog jump', `giraffe', and `plane',
and {\em manually annotated} saliency maps to mark the object regions
(see \figref{fig:dataset} and supplementary material for examples; full dataset to be publicly made available).
%
To normalize these images, we uniformly scale them so that their maximal dimension is 400 pixels long.
%For each image, if there is a non-ambiguous object with correct content matching with
%the query keyword and most part of the object is visible, we mark such an object region at pixel accuracy.
Some images in the dataset do not contain any salient object matching the keyword; we leave such images unlabeled.
Further, since partially occluded objects are less reliable for shape matching,
we only label object regions that are mostly un-occluded.
In the end, we got 6000+ images with pixel accurate ground truth segmentation
(see \figref{fig:dataset} for sample images and supplemental material for more statistics).
%\cmm{Pixel accurate labeling is quite expensive.
%For each image, we only have one human labeling.
%Although human labeling might not be accurate for some confusing examples,
%we argument that human performance is much better that current algorithms,
%thus such confusion could be ignored during evaluation.}
%
Note that Eitz \etal~\cite{11tvcg/eitz_sbir} introduce a benchmark dataset
for evaluating SBIR systems by annotating how well a given sketch and image pair match.
Our benchmark contains pixel-accurate  segmentation of the targets, when present,
thus allowing evaluation of corresponding segmentation algorithms.
Our dataset is $15 \times$ larger than previously largest public available
benchmark \cite{09cvpr/Achanta_FTSaliency} with pixel accuracy salient region annotation.
In contrast to the benchmark in \cite{09cvpr/Achanta_FTSaliency}, where salient regions are unambiguous,
clearly separated from the background, and often positioned near the image centers,
images in our proposed dataset are more challenging and represent typical cluttered real-world scenes.




%\begin{figure}[t!]
%  \centering
%  \includegraphics[width=\linewidth]{plots/SalMap2.pdf}
%  \caption{Precision-recall curves for naive thresholding of saliency maps.
%     S, G1, G2 represent single image saliency, group saliency after the 1st and 2nd
%     iterations, respectively.
%     Subscripts B, C, D, G, P represent groups of `butterfly', `coffee mug',
%     `dog jump', `giraffe', `plane', respectively.
%  } \label{fig:plots}
%\end{figure}


\subsection{Fixed thresholding of group saliency maps}


We threshold the saliency map with $T \in [0, 255]$ and compare the segmentation
results with ground truth labeling (see also \cite{09cvpr/Achanta_FTSaliency}).
The precision and recall curves in \figref{fig:plots} show that our group saliency
algorithm stabilizes after 2 iterations and significantly outperforms state-of-the-art
single image saliency detection method \cite{11cvpr/Cheng}.
%(see supplementary for comparison with other state-of-the-art saliency
%detection alternatives including FT~\cite{09cvpr/Achanta_FTSaliency}, SEG~\cite{10eccv/rahtu_segmenting}, and RC~\cite{11cvpr/Cheng}).




\subsection{Object of interest segmentation} \label{sec:EvalSeg}



%\begin{figure}[t!]
%  \centering
%  \begin{overpic}[width=.955\linewidth]{plots/CutCmp.pdf}
%       \small
%       \put(7,  1){butterfly}
%       \put(25, 1){coffee mug}
%       \put(45, 1){dog jump}
%       \put(66, 1){giraffe}
%       \put(86, 1){plane}
%  \end{overpic}
%  \caption{Comparison of $F_{\beta}$ for image groups using single image saliency
%    segmentation methods (FT~\cite{09cvpr/Achanta_FTSaliency},
%    SEG~\cite{10eccv/rahtu_segmenting}, RC~\cite{11cvpr/Cheng}) vs.
%    group saliency segmentation.
%  } \label{fig:plotsCut}
%\end{figure}



We also evaluate how accurately our algorithm extracts target objects from heterogeneous internet
images.
For images containing a target, we compare their pixel-level labeling with our group-saliency segmentation
according to precision, recall, and F-Measure, which is defined as:
\begin{equation}\label{equ:FMeasure}
    F_{\beta} = \frac{(1+\beta^2)Precision \times Recall}{\beta^2 \times Precision + Recall}.
\end{equation}
Note that for salient region segmentation,
precision is more important than recall \cite{09cvpr/Achanta_FTSaliency,11cvpr/Cheng},
since recall can trivially be $100\%$ by taking all image regions as targets.
For internet retrieval, precision is more important as a false detection
is undesirable over missing some good candidates among thousands of possibilities.
Hence, we use $\beta^2 = 0.3$ to weight precision more than recall for fair comparison with \sArt~methods
~\cite{09cvpr/Achanta_FTSaliency,11cvpr/Cheng,10eccv/rahtu_segmenting}.
%In the background of internet retrieval, precision is even more important than recall
%because that false report is more harmful than missing some of the correct images among thousands of them.
\figref{fig:plotsCut} illustrates the improvement due to group saliency segmentation.

Note that most of the nature-scene images contain multiple objects
and be associated with multiple text tags in by the search engine.
We use the group saliency based segmentation to extract saliency
object regions in images of the same group.
This allows us to extract objects in an image even when the text tags differ.
%Thus it is possible to provide the important shape descriptor for multi objects in the scene,
%enabling corresponding retrieval.

\begin{figure*}[t!]
    \centering
  \begin{overpic}[width=\linewidth, height=0.93\textheight]{Retrieval.jpg}
       \small
       \put(7,  1){butterfly}
       \put(19.5, 1){coffee mug}
       \put(35, 1){dog jump}
       \put(51, 1){giraffe}
       \put(69, 1){plane}
  \end{overpic}
    \caption{SBIR comparison.
         In each group from left to right, first column shows images downloaded from
        \href{http://www.flickr.com/}{Flickr} using the corresponding keyword;
        second column shows our retrieval results obtained by comparing user-input
        sketch with group saliency segmentation results;
        third column shows corresponding sketch based retrieval results using \SHoG.
        Two input sketches with their retrieval results are shown in (e).
    }
    \label{fig:SBIR_demo}
\end{figure*}



\begin{table}
    \small{\centering
    \begin{tabular}{c||c|c|c|c|c|c} \hline\hline
          \Rows{TPR (\%)}    & \multicolumn{3}{c|}{Among top 50} & \multicolumn{3}{c}{Among top 100}  \\ \cline{2-7}
                     & \ColInf & \ColInf \\ \hline
          {\small Bottle}   &  58 & 98 & 84 & 60 & 94 & 82  \\ \hline
          {\small Butterfly}&  28 & 54 & 36 & 28 & 50 & 40  \\ \hline
          {\small Cake}     &  56 & 84 & 78 & 55 & 81 & 79  \\ \hline
          {\small Cattle}   &  48 & 94 & 14 & 37 & 81 & 12  \\ \hline
          {\small Coffee mug}& 58 & 94 & 82 & 51 & 90 & 78  \\ \hline
          {\small Cow}      &  52 & 94 & 54 & 54 & 92 & 55  \\ \hline
          {\small Crow}     &  36 & 96 & 76 & 38 & 86 & 61  \\ \hline
          {\small Dog jump} &  56 & 92 & 74 & 55 & 85 & 73  \\ \hline
          {\small Eagle}    &  18 & 94 & 82 & 23 & 95 & 80  \\ \hline
          {\small Elephant} &  40 & 96 & 76 & 38 & 87 & 68  \\ \hline
          {\small Flag}     &  56 & 96 & 54 & 52 & 91 & 57  \\ \hline
          {\small Fox}      &  32 & 96 & 38 & 37 & 80 & 42  \\ \hline
          {\small Giraffe}  &  30 & 64 & 18 & 25 & 51 & 18  \\ \hline
          {\small iPhone}   &  48 & 96 & 68 & 43 & 90 & 54  \\ \hline
          {\small \small Jeep}     &  54 & 94 & 38 & 59 & 93 & 43  \\ \hline
          {\small \small Mailbox}  &  52 & 86 & 68 & 52 & 87 & 60  \\ \hline
          {\small Orange}   &  24 & 70 & 70 & 19 & 62 & 52  \\ \hline
          {\small Parrot}   &  58 & 90 & 60 & 56 & 79 & 57  \\ \hline
          {\small Pear}     &  64 & 84 & 82 & 63 & 82 & 81  \\ \hline
          {\small Plane}    &  44 & 94 & 90 & 48 & 93 & 91  \\ \hline
          {\small Sailboat} &  52 & 86 & 56 & 67 & 81 & 54  \\ \hline
          {\small Seagull}  &  56 & 92 & 88 & 59 & 90 & 86  \\ \hline
          {\small Sheep}    &  54 & 78 & 44 & 50 & 81 & 40  \\ \hline
          {\small Snail}    &  62 & 94 & 60 & 63 & 87 & 61  \\ \hline
          {\small Strawberry}& 50 & 70 & 76 & 53 & 69 & 67  \\ \hline
          {\small Swallow}  &  24 & 76 & 74 & 31 & 66 & 68  \\ \hline
          {\small Tank}     &  48 & 88 & 56 & 44 & 84 & 45  \\ \hline
          {\small Tortoise} &  48 & 62 & 46 & 48 & 62 & 45  \\ \hline
          {\small Wolf}     &  24 & 50 & 30 & 25 & 55 & 29  \\ \hline
          {\small Zebra}    &  30 & 66 & 28 & 25 & 56 & 31  \\ \hline
          \textbf{Average}  & \textbf{46.0}&\textbf{84.2}&\textbf{60.0}&\textbf{45.3}&\textbf{79.3}&\textbf{57.0} \\ \hline
          \hline
    \end{tabular}}
    \caption{True positive ratios (TPR) among top $50$ and $100$ retrievals
        using Flickr, our method, and \SHoG$ $ for 30 different categories.
    }\label{tab:statistic}
\end{table}



\subsection{Sketch based image retrieval}


As an application, we compare our retrieval algorithm with \sArt$ $  SBIR
proposed by Eitz \etal~\cite{11tvcg/eitz_sbir} (using author implementations).
Our method explicitly extracts object of interest regions from images,
thus enabling us to exploit the power of existing shape matching techniques.
For heterogeneous internet images, the combination of group-saliency segmentation and shape matching
effectively selects good images containing target objects, leading to improved results
(see \tabref{tab:statistic}, \figref{fig:SBIR_demo}, and supplementary material).
%
We leave exploring benefits of hybrid systems using additional attributes including appearance~\cite{csur08/datta},
local features~\cite{11tvcg/eitz_sbir}, or additional lines~\cite{cao2010mindfinder} to future research.

%Although the group saliency segmentation don't produce very high accuracy results
%for noisy internet images,
%the shape matching algorithms helps to effectively select those good results,
%leading to exciting results as shown in \tabref{tab:statistic}.


We pre-process each image in the database by performing single image unsupervised segmentation
(about 100 images per minute).
Further, we  preprocess each category using a representative sketch to initialize a good
appearance learning and unsupervised segmentation of the object.
%
In \figref{fig:SBIR_demo}(e), we show retrieval results for `plane' with two {\em different} input sketches,
 for which results for the second sketches just have to compare salient shapes generated with the help of
the first sketch.
Our results contain explicit region information allowing input sketches to retrieve
results with more relevant pose.

Note that currently our system expect users to supply both keywords and sketches.
%
Existing shape matching techniques,
which are able to effectively select high quality matchings
from shapes with clean background, even with very rough sketches
(e.g., \sArt method \cite{yang2009locally}), could achieve $93.3\%$
accuracy in the very challenging MPEG7 shape dataset.
Once the user inputs a rough sketch to help distinguish between
desired object and irrelevant region shapes,
it would help us to get useful global appearance information.
The low correlation between shape feature and appearance statistics allows
us to reuse the input sketch, learned appearance, and segmented regions.
%
Recent advances in human object sketch classifications~\cite{12TOG/Eitz} can potentially
be used in conjunction with our system towards a keyword-free retrieval interface,
which can be attractive for gesture-based devices.
%
At runtime, we only compare a new user sketch with object shapes using the cascade filtering
process (see \secref{sec:groupSaliency}) taking less than 1 second to handle an initial
retrieved set of $3,000$ images.
For larger databases, efficient retrieval algorithms using shape
context~\cite{01cvpr_MoriBM_SCRetrieval} may be useful.
Our method only segments the most salient object region from each image
and perform retrieval based on that object region.
Since there are a huge number of internet images,
we are mainly focused on quality of the top ranked
results rather than the recall of every image.






%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We introduced a method to exploit correlations across internet images within
same categories to achieve superior salient object segmentation and image retrieval.
%presented to make use of cross relations between internet images of the same categories.
Starting from a simple user sketch, we estimate high quality image labeling to
build appearance models for target image regions and their backgrounds.
These appearance models are in turn used to improve saliency detection and image segmentation.
We introduced a benchmark consisting of 6000+ pixel-accurate labeled dataset initially obtained by querying
keywords from \href{http://www.flickr.com/}{Flickr} and use it to demonstrate that our proposed method produces
high quality saliency maps and segmentation, with potential application to SBIR.
%
Our approach makes use of the powerful user sketch information to select good segmentation
candidates for getting global appearance information (see \secref{sec:GlobalStatis}).
This selection process avoids error accumulative problems which typically exist in
iterative segmentation transfer methods \cite{kuettel2012segmentation},
resulting in the consistent result improvements observed in our experiments.


In the future, we plan to learn additional texture and shape statistics~\cite{cvpr07/charpiat}
to further improve the segmentation. We also plan to investigate efficient
shape indexing algorithms~\cite{08CVPR/peter_shape,10TOM/chellappa_efficient}
and GPU speed up \cite{schmid2011gpu} for increased efficiency.


\begin{acknowledgements}
We would like to thank the anonymous reviewers for their constructive comments.
This research was supported by the 973 Program (2011CB302205),
the 863 Program (2009AA01Z327), the Key Project of S\&T (2011ZX01042-001-002),
and NSFC (U0735001).
Ming-Ming Cheng was funded by Google PhD fellowship, IBM PhD fellowship,
and New PhD Researcher Award (Ministry of Edu., CN).
\end{acknowledgements}


%\bibliographystyle{spbasic}      % basic style, author-year citations
\bibliographystyle{spmpsci}      % mathematics and physical sciences
%\bibliographystyle{spphys}       % APS-like style for physics
%\bibliography{}   % name your BibTeX data base
\bibliography{GroupSaliency}


\end{document}
% end of file template.tex

