Publications

Introduction: Math anxiety is a common challenge for young students, often impairing learning and engagement. Tangible and extended reality (XR) technologies offer opportunities to make learning more interactive, collaborative, and less intimidating. Objective: This work proposes XR4Math, a low-cost tangible XR system designed to reduce math anxiety through personalized and engaging activities. Methodology: The system synthesizes principles from embodied cognition, collaboration, and multimodal design. Results: We demonstrate its potential through two proof-of-concept applications: ExpressionXR, a tangible game for solving arithmetic expressions, and PuzzleXR, a geometric construction puzzle.

@inproceedings{ihc,
 author = {Leonardo M. de Moraes and Gustavo G. Souto and Carlos Morimoto},
 title = { Towards Low-Cost Tangible Extended Reality for Mathematics Education},
 booktitle = {Anais do XXIV Simpósio Brasileiro sobre Fatores Humanos em Sistemas Computacionais},
 location = {Belo Horizonte/MG},
 year = {2025},
 keywords = {},
 issn = {0000-0000},
 pages = {518--534},
 publisher = {SBC},
 address = {Porto Alegre, RS, Brasil},
 doi = {10.5753/ihc.2025.10835},
 url = {https://sol.sbc.org.br/index.php/ihc/article/view/37675}
}

Web browsing is essential for modern education, supporting everything from self-directed study to academic research. However, traditional web interfaces are designed for keyboards and mice, and touch screens creating accessibility barriers for users with motor impairments. Modern Head-Mounted Displays (HMDs), which lack these conventional input devices but often include eye-tracking technology, make gaze-based interaction in XR a promising alternative due to its immersive experience. For a gaze-based XR web browser to be viable, it must be efficient and provide a good user experience. To this end, this paper proposes XRBars, a system that leverages GazeBars to improve accessibility to online educational resources in XR.

@inproceedings{imxw,
 author = {João Nogueira and Carlos Morimoto},
 title = { XRBars: Fast and Reliable Multiple Choice Selection by Gaze in XR},
 booktitle = {Proceedings of the ACM International Conference on Interactive Media Experiences Workshops},
 location = {Niterói/RJ},
 year = {2025},
 keywords = {},
 issn = {0000-0000},
 pages = {155--159},
 publisher = {SBC},
 address = {Porto Alegre, RS, Brasil},
 doi = {10.5753/imxw.2025.2085},
 url = {https://sol.sbc.org.br/index.php/imxw/article/view/35246}
}

Due to the low accuracy and noisy data of common eye trackers, virtual gaze-activated targets used for gaze interaction tend to be large, so they can be easily fixated and selected. Therefore, a full-size virtual keyboard with 30 to 40 keys may take a large portion of the screen space. If one simply reduces the keyboard size into a higher density interface, selections would become unreliable. In this paper we present XTar, an expandable target interface that uses an entropy-based algorithm to expand and improve the robustness of target selection in high density interfaces. Results from a pilot user experiment show that typing using a dwell-time linear XTar keyboard is as fast as a 3 row QWERTY keyboard with fixed size keys, with similar low error rate, using about 1/3 of the QWERTY keyboard area.

@article{10.1145/3729408,
author = {Gonzales, Candy T and Morimoto, Carlos H},
title = {XTar: entropy-based target expansion for high density gaze-based interfaces},
year = {2025},
issue_date = {June 2025},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {8},
number = {2},
url = {https://doi.org/10.1145/3729408},
doi = {10.1145/3729408},
abstract = {Due to the low accuracy and noisy data of common eye trackers, virtual gaze-activated targets used for gaze interaction tend to be large, so they can be easily fixated and selected. Therefore, a full-size virtual keyboard with 30 to 40 keys may take a large portion of the screen space. If one simply reduces the keyboard size into a higher density interface, selections would become unreliable. In this paper we present XTar, an expandable target interface that uses an entropy-based algorithm to expand and improve the robustness of target selection in high density interfaces. Results from a pilot user experiment show that typing using a dwell-time linear XTar keyboard is as fast as a 3 row QWERTY keyboard with fixed size keys, with similar low error rate, using about 1/3 of the QWERTY keyboard area.},
journal = {Proc. ACM Comput. Graph. Interact. Tech.},
month = may,
articleno = {28},
numpages = {15},
keywords = {gaze selection, target expansion, entropy, eye tracking, eye gaze-based interfaces}
}

The visual pursuit of moving targets is a natural behaviour that has been exploited in, for example, medical diagnosis, law enforcement, and human computer interaction. Most proposed algorithms to detect this behaviour are based on some kind of motion similarity metric that assumes small or no distortion between the trajectory described by the target being pursued and the sensor measurements. We propose a novel algorithm based on 1D Convolutional Neural Networks (1D-CNNs), and investigate the performance of the 1D-CNN against 4 state-of-the-art similarity based algorithms (SAs), using a novel dataset containing data from 10 participants. Their performances are evaluated using two trajectory shapes (circle and square), two target speeds (120°/s and 240°/s), and three window sizes (0.5, 1.0 and 1.5 s). All algorithms have been trained or optimized to maximize the Matthew's Correlation Coefficient (MCC). Our results show that the 1D-CNN outperforms the SAs in all cases, requiring smaller window sizes to robustly detect the pursuits.

@article{CARNEIRO202445,
title = {Detection of visual pursuits using 1D convolutional neural networks},
journal = {Pattern Recognition Letters},
volume = {179},
pages = {45-51},
year = {2024},
issn = {0167-8655},
doi = {https://doi.org/10.1016/j.patrec.2024.01.020},
url = {https://www.sciencedirect.com/science/article/pii/S0167865524000163},
author = {Alex Torquato S. Carneiro and Flavio Luiz Coutinho and Carlos H. Morimoto},
keywords = {Eye tracking, Eye gaze pursuit, Convolutional neural networks, Eye movements classification},
abstract = {The visual pursuit of moving targets is a natural behaviour that has been exploited in, for example, medical diagnosis, law enforcement, and human computer interaction. Most proposed algorithms to detect this behaviour are based on some kind of motion similarity metric that assumes small or no distortion between the trajectory described by the target being pursued and the sensor measurements. We propose a novel algorithm based on 1D Convolutional Neural Networks (1D-CNNs), and investigate the performance of the 1D-CNN against 4 state-of-the-art similarity based algorithms (SAs), using a novel dataset containing data from 10 participants. Their performances are evaluated using two trajectory shapes (circle and square), two target speeds (120°/s and 240°/s), and three window sizes (0.5, 1.0 and 1.5 s). All algorithms have been trained or optimized to maximize the Matthew's Correlation Coefficient (MCC). Our results show that the 1D-CNN outperforms the SAs in all cases, requiring smaller window sizes to robustly detect the pursuits.}
}

Web accessibility and digital inclusion are still one of the great research challenges in computer science. For people with severe motor disabilities, eye-gaze interaction enables hands-free operation of graphical interfaces. Nonetheless, gaze tracking devices typically require frequent user calibration that weakens the overall user experience and performance. In this paper we propose EyePursuitLinks, an eye-pursuit based technique that allows people with severe motor disabilities to browse the web by following multiple moving targets corresponding to the links they want to follow. The main contribution of this paper is the use of Smart Targets (ST) to select potentially large number of hyperlinks within a relatively small area. We have performed a user experiment with 10 volunteers to evaluate the performance of the Smart Targets selection against conventional pursuit selection mechanism using 4, 8, and 16 simultaneous targets. Our results show that the use of ST is significantly more robust than the conventional method for larger number of targets.

@inproceedings{10.1145/3617023.3617058,
author = {Carneiro, Alex Torquato Souza and Gonzales, Candy Veronica Tenorio and Morimoto, Carlos Hitoshi},
title = {EyePursuitLinks - an Eye-pursuit Based Interface for Web Browsing Using Smart Targets},
year = {2023},
isbn = {9798400709081},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3617023.3617058},
doi = {10.1145/3617023.3617058},
abstract = {Web accessibility and digital inclusion are still one of the great research challenges in computer science. For people with severe motor disabilities, eye-gaze interaction enables hands-free operation of graphical interfaces. Nonetheless, gaze tracking devices typically require frequent user calibration that weakens the overall user experience and performance. In this paper we propose EyePursuitLinks, an eye-pursuit based technique that allows people with severe motor disabilities to browse the web by following multiple moving targets corresponding to the links they want to follow. The main contribution of this paper is the use of Smart Targets (ST) to select potentially large number of hyperlinks within a relatively small area. We have performed a user experiment with 10 volunteers to evaluate the performance of the Smart Targets selection against conventional pursuit selection mechanism using 4, 8, and 16 simultaneous targets. Our results show that the use of ST is significantly more robust than the conventional method for larger number of targets.},
booktitle = {Proceedings of the 29th Brazilian Symposium on Multimedia and the Web},
pages = {16–24},
numpages = {9},
keywords = {Smooth pursuits, Responsive interfaces, Probabilistic algorithm, Eye-gaze interaction},
location = {, Ribeir~{a}o Preto, Brazil, },
series = {WebMedia '23}
}

The simultaneous classification of the three most basic eye-movement patterns is known as the ternary eye-movement classification problem (3EMCP). Dynamic, interactive real-time applications that must instantly adjust or respond to certain eye behaviors would highly benefit from accurate, robust, fast, and low-latency classification methods. Recent developments based on 1D-CNN-BiLSTM and TCN architectures have demonstrated to be more accurate and robust than previous solutions, but solely considering offline applications. In this paper, we propose a TCN classifier for the 3EMCP, adapted to online applications, that does not require look-ahead buffers. We introduce a new lightweight preprocessing technique that allows the TCN to make real-time predictions at about 500 Hz with low latency using commodity hardware. We evaluate the TCN performance against other two deep neural models: a CNN-LSTM and a CNN-BiLSTM, also adapted to online classification. Furthermore, we compare the performance of the deep neural models against a lightweight real-time Bayesian classifier (I-BDT). Our results, considering two publicly available datasets, show that the proposed TCN model consistently outperforms other methods for all classes. The results also show that, though it is possible to achieve reasonable accuracy levels with zero-length look ahead, the performance of all methods improve with the use of look-ahead information. The codebase, pre-trained models, and datasets are available at https://github.com/elmadjian/OEMC.

@Article{Elmadjian2022,
author={Elmadjian, Carlos
and Gonzales, Candy
and Costa, Rodrigo Lima da
and Morimoto, Carlos H.},
title={Online eye-movement classification with temporal convolutional networks},
journal={Behavior Research Methods},
year={2022},
month={Oct},
day={11},
abstract={The simultaneous classification of the three most basic eye-movement patterns is known as the ternary eye-movement classification problem (3EMCP). Dynamic, interactive real-time applications that must instantly adjust or respond to certain eye behaviors would highly benefit from accurate, robust, fast, and low-latency classification methods. Recent developments based on 1D-CNN-BiLSTM and TCN architectures have demonstrated to be more accurate and robust than previous solutions, but solely considering offline applications. In this paper, we propose a TCN classifier for the 3EMCP, adapted to online applications, that does not require look-ahead buffers. We introduce a new lightweight preprocessing technique that allows the TCN to make real-time predictions at about 500 Hz with low latency using commodity hardware. We evaluate the TCN performance against other two deep neural models: a CNN-LSTM and a CNN-BiLSTM, also adapted to online classification. Furthermore, we compare the performance of the deep neural models against a lightweight real-time Bayesian classifier (I-BDT). Our results, considering two publicly available datasets, show that the proposed TCN model consistently outperforms other methods for all classes. The results also show that, though it is possible to achieve reasonable accuracy levels with zero-length look ahead, the performance of all methods improve with the use of look-ahead information. The codebase, pre-trained models, and datasets are available at https://github.com/elmadjian/OEMC.},
issn={1554-3528},
doi={10.3758/s13428-022-01978-2},
url={https://doi.org/10.3758/s13428-022-01978-2}
}

In this course you will learn the basics to conduct scientific research in Human ComputerInteraction (HCI), i.e., to plan, conduct, and analyze the results of an experiment withusers interacting with some computational device. These experiments are fundamentalfor the innovation and development of interactive products. The course blends theory andpractice, beginning with a brief introduction to HCI and the interaction design processto motivate the need to conduct user experiments. To conduct an experiment, you will berequired to define your methodology and follow a rigorous statistical analysis to validateyour results. At the end you will conduct a simple experiment to apply these concepts andevaluate a model that helps predict user performance in pointing tasks.

@incollection{Morimoto:JAI2021,
  author      = "Morimoto, C. H. and Diaz-Tula, A.", 
  title       = "Métodos Experimentais em Interação Humano Computador",
  editor      = "Andrade, A. M. S. and Wazlawick, R. S.",
  booktitle   = "Jornada de Atualização em Informática 2021",
  publisher   = "SBC",
  address     = "Porto Alegre, RS",
  year        = 2021,
  pages       = "1-43",
  chapter     = 2,
  doi = "https://doi.org/10.5753/sbc.6757.3.2",
  url = "https://sol.sbc.org.br/livros/index.php/sbc/catalog/book/67",
}

This paper introduces a bi-modal typing interface, HGaze Typing, which combines the simplicity of head gestures with the speed of gaze inputs to provide efficient and comfortable dwell-free text entry. HGaze Typing uses gaze path information to compute candidate words and allows explicit activation of common text entry commands, such as selection, deletion, and revision, by using head gestures (nodding, shaking, and tilting). By adding a head-based input channel, HGaze Typing reduces the size of the screen regions for cancel/deletion buttons and the word candidate list, which are required by most eye-typing interfaces. A user study finds HGaze Typing outperforms a dwell-time-based keyboard in efficacy and user satisfaction. The results demonstrate that the proposed method of integrating gaze and head-movement inputs can serve as an effective interface for text entry and is robust to unintended selections.

@inbook{10.1145/3448017.3457379,
author = {Feng, Wenxin and Zou, Jiangnan and Kurauchi, Andrew and Morimoto, Carlos H and Betke, Margrit},
title = {HGaze Typing: Head-Gesture Assisted Gaze Typing},
year = {2021},
isbn = {9781450383448},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3448017.3457379},
abstract = { This paper introduces a bi-modal typing interface, HGaze Typing, which combines the simplicity of head gestures with the speed of gaze inputs to provide efficient and comfortable dwell-free text entry. HGaze Typing uses gaze path information to compute candidate words and allows explicit activation of common text entry commands, such as selection, deletion, and revision, by using head gestures (nodding, shaking, and tilting). By adding a head-based input channel, HGaze Typing reduces the size of the screen regions for cancel/deletion buttons and the word candidate list, which are required by most eye-typing interfaces. A user study finds HGaze Typing outperforms a dwell-time-based keyboard in efficacy and user satisfaction. The results demonstrate that the proposed method of integrating gaze and head-movement inputs can serve as an effective interface for text entry and is robust to unintended selections.},
booktitle = {ACM ETRA 2021 -- Best Paper Award},
articleno = {11},
numpages = {11}
}

Motion correlation interfaces are those that present targets moving in different patterns, which the user can select by matching their motion. In this paper, we re-formulate the task of target selection as a probabilistic inference problem. We demonstrate that previous interaction techniques can be modelled using a Bayesian approach and that how modelling the selection task as transmission of information can help us make explicit the assumptions behind similarity measures. We propose ways of incorporating uncertainty into the decision-making process and demonstrate how the concept of entropy can illuminate the measurement of the quality of a design. We apply these techniques in a case study and suggest guidelines for future work.

@inproceedings{10.1145/3411764.3445184,
author = {Velloso, Eduardo and Morimoto, Carlos H},
title = {A Probabilistic Interpretation of Motion Correlation Selection Techniques},
year = {2021},
isbn = {9781450380966},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3411764.3445184},
doi = {10.1145/3411764.3445184},
abstract = { Motion correlation interfaces are those that present targets moving in different patterns, which the user can select by matching their motion. In this paper, we re-formulate the task of target selection as a probabilistic inference problem. We demonstrate that previous interaction techniques can be modelled using a Bayesian approach and that how modelling the selection task as transmission of information can help us make explicit the assumptions behind similarity measures. We propose ways of incorporating uncertainty into the decision-making process and demonstrate how the concept of entropy can illuminate the measurement of the quality of a design. We apply these techniques in a case study and suggest guidelines for future work.},
booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},
articleno = {285},
numpages = {13},
keywords = {gaze interaction, probabilistic input, gestures, pursuits, computational interaction, motion correlation},
location = {Yokohama, Japan},
series = {CHI '21}
}

Imagine an application that requires constant configuration changes, such as modifying the brush type in a drawing application. Typically, options are hierarchically organized in menu bars that the user must navigate, sometimes through several levels, to select the desired mode. An alternative to reduce hand motion is the use of multimodal techniques such as textit{gaze-touch}, that combines gaze pointing with mechanical selection. In this paper, we introduce GazeBar, a novel multimodal gaze interaction technique that uses gaze paths as a combined pointing and selection mechanism. The idea behind GazeBar is to maximize the interaction flow by reducing "safety" mechanisms (such as clicking) under certain circumstances. We present GazeBar's design and demonstrate it using a digital drawing application prototype. Advantages and disadvantages of GazeBar are discussed based on a user performance model.

@inbook{10.1145/3411763.3451703,
author = {Elmadjian, Carlos and Morimoto, Carlos H},
title = {GazeBar: Exploiting the Midas Touch in Gaze Interaction},
year = {2021},
isbn = {9781450380959},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3411763.3451703},
abstract = { Imagine an application that requires constant configuration changes, such as modifying the brush type in a drawing application. Typically, options are hierarchically organized in menu bars that the user must navigate, sometimes through several levels, to select the desired mode. An alternative to reduce hand motion is the use of multimodal techniques such as gaze-touch, that combines gaze pointing with mechanical selection. In this paper, we introduce GazeBar, a novel multimodal gaze interaction technique that uses gaze paths as a combined pointing and selection mechanism. The idea behind GazeBar is to maximize the interaction flow by reducing ”safety” mechanisms (such as clicking) under certain circumstances. We present GazeBar’s design and demonstrate it using a digital drawing application prototype. Advantages and disadvantages of GazeBar are discussed based on a user performance model.},
booktitle = {Extended Abstracts of the 2021 CHI Conference on Human Factors in Computing Systems},
articleno = {248},
numpages = {7}
}

Recently, deep learning approaches have been proposed to detect eye movements such as fixations, saccades, and smooth pursuits from eye tracking data. These are textit{end-to-end} methods that have shown to surpass traditional ones, requiring no textit{ad hoc} parameters. In this work we propose the use of temporal convolutional networks (TCNs) for automated eye movement classification and investigate the influence of feature space, scale, and context window sizes on the classification results. We evaluated the performance of TCNs against a state-of-the-art 1D-CNN-BLSTM model using GazeCom, a public available dataset. Our results show that TCNs can outperform the 1D-CNN-BLSTM, achieving an F-score of 94.2% for fixations, 89.9% for saccades, and 73.7% for smooth pursuits on sample level, and 89.6%, 94.3%, and 60.2% on event level. We also state the advantages of TCNs over sequential networks for this problem, and how these scores can be further improved by feature space extension. keywords{Eye movement classification and Temporal convolutional networks and Feature selection.

@inproceedings{Elmadjian:2020,
author = {Elmadjian, Carlos and Gonzales, Candy and Morimoto, Carlos H.},
title = {Eye Movement Classification With Temporal Convolutional Networks},
booktitle = {Proceedings of the Workshop on Eye Tracking Techniques, Applications and Challenges},
series = {ETTAC '20},
year = {2020},
url = {https://vision.unipv.it/ettac2020/program.html},
}

Swipe-based methods for text entry by gaze allow users to swipe through the letters of a word by gaze, analogous to how they can swipe with a finger on a touchscreen keyboard. Two challenges for these methods are: (1) gaze paths do not possess clear start and end positions, and (2) it is difficult to design text editing features. We introduce Swipe&Switch, a text-entry interface that uses swiping and switching to improve gaze-based interaction. The interface contains three context regions, and detects the start/end of a gesture and emits text editing commands (e.g., word insertion, deletion) when a user switches focus between these regions. A user study showed that Swipe&Switch provides a better user experience and higher text entry rate over a baseline, EyeSwipe.

@inproceedings{Kurauchi:UIST2020,
author = {Kurauchi, Andrew and Feng, Wenxin and Joshi, Ajjen and Morimoto, Carlos H. and Betke, Margrit},
title = {Swipe&Switch: Text Entry Using Gaze Paths and Context Switching},
year = {2020},
isbn = {9781450375153},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3379350.3416193},
doi = {10.1145/3379350.3416193},
abstract = {Swipe-based methods for text entry by gaze allow users to swipe through the letters of a word by gaze, analogous to how they can swipe with a finger on a touchscreen keyboard. Two challenges for these methods are: (1) gaze paths do not possess clear start and end positions, and (2) it is difficult to design text editing features. We introduce Swipe&Switch, a text-entry interface that uses swiping and switching to improve gaze-based interaction. The interface contains three context regions, and detects the start/end of a gesture and emits text editing commands (e.g., word insertion, deletion) when a user switches focus between these regions. A user study showed that Swipe&Switch provides a better user experience and higher text entry rate over a baseline, EyeSwipe.},
booktitle = {Adjunct Publication of the 33rd Annual ACM Symposium on User Interface Software and Technology},
pages = {84–86},
numpages = {3},
keywords = {gesture-based typing, text entry, gaze swiping, eye tracking, eye typing},
location = {Virtual Event, USA},
series = {UIST '20 Adjunct}
}

The use of a single uncalibrated camera is desirable for eye tracking to reduce the overall complexity and cost of the system. Quite often, at least one external light source is used to enhance image quality and generate a corneal reflection used as a reference point to estimate the point-of-gaze (PoG). Though the use of more than one light source has shown to enhance accuracy and robustness to head motion, it is unlikely that all corneal reflections appear in the eye images during natural eye movements. In this paper, we introduce the Screen-Light Decomposition (SLD) framework as a generalized model for PoG estimation using a single uncalibrated camera and a variable number of light sources. SLD synthesizes existing uncalibrated video-based eye trackers and can be used as a modeling tool to compare and design eye trackers. We have used the framework to design a novel eye-tracking technique, called SAGE, for single normalized space adaptive gaze estimation, that can gracefully degrade the gaze tracker performance when one or more corneal reflections are not detected, even during the calibration procedure. Results from an user experiment are presented to demonstrate its improved performance over other designs.

@article{Morimoto:JMIV2020,
  title = {Screen-Light Decomposition Framework for Point-of-Gaze Estimation Using a Single Uncalibrated Camera and Multiple Light Sources},
  issn = {1573-7683},
  url = {https://doi.org/10.1007/s10851-020-00947-8},
  doi = {10.1007/s10851-020-00947-8},
  journal = {Journal of Mathematical Imaging and Vision},
 publisher={Springer},
 date = {2020-02-15},
volume={62},
number={4},
pages={586-605},
year={2020},
  author = {Morimoto, Carlos H. and Coutinho, Flávio L. and Hansen, Dan W.}
}

As our lives get more deeply submerged in digital format, ubiquitous access to sensitive data requires more secure and efficient user authentication procedures. Methods that solely relied on password entry were lately enhanced with the use of biometrics. Yet, these techniques can still be tricked by, for example, recordings of the face, voice, and fingerprint cloning. In this paper we introduce PursuitPass, a compact, robust, and efficient visual pursuit-based authentication system. PursuitPass is a user calibration-free method that requires the user to enter a password by visually pursuing moving targets on a small screen, such as a public ATM or a personal mobile phone. Because eye movements are used as input, passwords are better protected against shoulder surfing. Also, since targets can potentially move in unpredictable ways, it naturally imposes a liveness feature that cannot be counterfeited by recordings of the eyes. We investigated four pattern-matching algorithms to match visual pursuit user data with the movement of the targets. Two experiments were conducted. The first experiment aimed to define the best performing matching algorithm and configuration for PursuitPass. The second experiment aimed to evaluate the performance of our prototype. PursuitPass achieved a 96.82% accuracy with an average time of 10.42 s on a series of 4-digit PIN entry trials.

@inproceedings{Carneiro:Sibgrapi19,
author = {Carneiro, A. T. S. and Elmadjian, C. E. and Gonzalez, C. and Coutinho, F. L. and Morimoto, C. H.},
title = {PursuitPass: A Visual Pursuit-Based User Authentication System},
year = {2019},
month = {October},
booktitle = {Electronic Proceedings of the 31st Conference on Graphics, Patterns and Images (SIBGRAPI'19)},
address = {Rio de Janeiro, RJ, Brazil},
pages = {226-233},
url = {https://conferences.computer.org/sibgrapi/2019/pdfs/SIBGRAPI2019-44CNA7QI0AIw6YsYtKI4QT/49ZoMetKHZxsy7KBLpEszo/12bz4P9G7h8oriSq0rgDwz.pdf},
doi={10.1109/SIBGRAPI.2019.00038},
}

Virtual makeup systems allow users to try makeup on remotely, without wasting products or spending time cleaning up later. Some virtual makeup systems just show the final makeup result on the user’s face, while other systems allow some interaction when applying makeup on a photo. In this paper we introduce an augmented reality system that allows users to apply virtual makeup directly on their face using a physical applicator, simulating a virtual mirror experience. Facial features are detected and tracked using an RGBD camera and mapped to a normalized 2D facial mesh composed of 124 triangles. Finger touches on the face are also detected on the RGBD video stream, and used to store the applied makeup texture representation on the corresponding 2D facial triangle. Rendering the face with the virtual makeup is performed by back-projecting the makeup stored on the facial mesh to the image captured by the camera. Our initial prototype demonstrates the feasibility of our technique, that detects touches very accurately (about 2.2 mm), and that achieves real-time interactive performance (about 15 fps) when tracking and rendering makeup using a regular PC and an Intel RealSense RGBD camera.

@inproceedings{Borges:Sibgrapi19,
author = {Borges, A. F. S. and Morimoto, C. H.},
title = {A Virtual Makeup Augmented Reality System},
year = {2019},
month = {October},
booktitle = {Electronic Proceedings of the  21st Symposium on Virtual and Augmented Reality (SVR)},
address = {Rio de Janeiro, RJ, Brazil},
pages = {34-42},
url = {https://tinyurl.com/vc2eb3d},
doi={10.1109/SVR.2019.00022},
isbn={978-1-7281-5434-3},
}

Despite recent developments in eye tracking technology, mobile eye trackers (ET) are still expensive devices limited to a few hundred samples per second. High speed ETs (closer to 1 KHz) can provide improved flexibility for data filtering and more reliable event detection. To address these challenges, we present the Stroboscopic Catadioptric Eye Tracking (SCET) system, a novel approach for mobile ET based on rolling shutter cameras and stroboscopic structured infrared lighting. SCET proposes a geometric model where the cornea acts as a spherical mirror in a catadioptric system, changing the projection as it moves. Calibration methods for the geometry of the system and for the gaze estimation are presented. Instead of tracking common eye features, such as the pupil center, we track multiple glints on the cornea. By carefully adjusting the camera exposure and the lighting period, we show how one image frame can be divided into several bands to increase the temporal resolution of the gaze estimates. We assess the model in a simulated environment and also describe a prototype implementation that demonstrates the feasibility of SCET, which we envision as a step further in the direction of a mobile, robust, affordable, and high-speed eye tracker.

@inproceedings{Borsato:2019:TLC:3314111.3319841,
 author = {Borsato, Frank H. and Morimoto, Carlos H.},
 title = {Towards a Low Cost and High Speed Mobile Eye Tracker},
 booktitle = {Proceedings of the 11th ACM Symposium on Eye Tracking Research & Applications},
 series = {ETRA '19},
 year = {2019},
 isbn = {978-1-4503-6709-7},
 location = {Denver, Colorado},
 pages = {16:1--16:9},
 articleno = {16},
 numpages = {9},
 url = {http://doi.acm.org/10.1145/3314111.3319841},
 doi = {10.1145/3314111.3319841},
 acmid = {3319841},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {catadioptric system, mobile eye-tracking, rolling shutter, stroboscopic lighting},
}

Eye tracking (ET) for gaze interaction in wearable computing imposes harder constraints on computational efficiency and illumination conditions than remote ET. In this paper we present xSDL, an extended temporal support computer vision algorithm for accurate, robust, and efficient pupil detection and gaze estimation. The robustness and efficiency of xSDL partly come from the use of stroboscopic differential lighting (SDL), an extension of the differential lighting pupil detection technique developed in the 90’s. Due to the erratic behavior of eye movements, traditional computer vision tracking techniques (such as Kalman filters) do not perform well, so most ET techniques simply detect some eye feature (such as the pupil center) at every frame. Extended temporal support uses keyframes selected during eye fixations and a simple translation model of the pupil to further improve the computational performance of SDL. A prototype composed of two independent acquisition systems was developed to evaluate the performance of xSDL and other four state-of-the-art ET techniques under similar conditions. Our results show that xSDL outperforms those four algorithms, both in speed (close to 2000 Hz using 240 line frames) and accuracy.

@Article{Borsato2019,
author="Borsato, Frank H.
and Diaz-Tula, Antonio
and Morimoto, Carlos H.",
title="xSDL: stroboscopic differential lighting eye tracker with extended temporal support",
journal="Machine Vision and Applications",
year="2019",
month="Apr",
day="15",
volume={30},
number={4},
abstract="Eye tracking (ET) for gaze interaction in wearable computing imposes harder constraints on computational efficiency and illumination conditions than remote ET. In this paper we present xSDL, an extended temporal support computer vision algorithm for accurate, robust, and efficient pupil detection and gaze estimation. The robustness and efficiency of xSDL partly come from the use of stroboscopic differential lighting (SDL), an extension of the differential lighting pupil detection technique developed in the 90's. Due to the erratic behavior of eye movements, traditional computer vision tracking techniques (such as Kalman filters) do not perform well, so most ET techniques simply detect some eye feature (such as the pupil center) at every frame. Extended temporal support uses keyframes selected during eye fixations and a simple translation model of the pupil to further improve the computational performance of SDL. A prototype composed of two independent acquisition systems was developed to evaluate the performance of xSDL and other four state-of-the-art ET techniques under similar conditions. Our results show that xSDL outperforms those four algorithms, both in speed (close to 2000 Hz using 240 line frames) and accuracy.",
issn="1432-1769",
doi="10.1007/s00138-019-01022-y",
url="https://doi.org/10.1007/s00138-019-01022-y",
pages="689-703",
}

Structured lighting (SL) image processing relies on the generation of known illumination patterns synchronized with the camera frame rate and is commonly implemented using syncing capable cameras. In general, such cameras employ global shutters, that exposes the whole frame at once. However, most modern digital cameras use rolling shutters, which expose each line at different intervals, impairing most structured lighting applications. In this paper we introduce an asynchronous SL technique that can be used by any rolling shutter digital camera. While the use of stroboscopic illumination partially solves for the line exposure shift, the phase difference between the camera and lighting clocks results in stripe artifacts that move vertically in the video stream. These stripes are detected and tracked using a Kalman filter. Two asynchronous stroboscopic SL methods are proposed. The first method, image differencing, minimizes the stripe artifacts. The second method, image compositing, completely removes the artifacts. We demonstrate the use of the asynchronous differential lighting technique in a pupil detector using a low-cost high-speed camera with no synchronization means, with the lighting running independently at a higher, unknown frequency to the application.

@inproceedings{Borsato:Sibgrapi18, 
author = {Borsato, Frank H. and Morimoto, Carlos H.}, 
title = {Asynchronous stroboscopic structured lighting image processing using low-cost cameras},
year = {2018}, 
month = {November},
booktitle = {Electronic Proceedings of the 31st Conference on Graphics, Patterns and Images (SIBGRAPI'18)},
address = {Foz do Iguaçu, PR Brazil}, 
url = {http://urlib.net/rep/8JMKD3MGPAW/3RPADUS?ibiurl.backgroundlanguage=en},
}

Second order polynomials are commonly used for estimating the point-of-gaze in head- mounted eye trackers. Studies in remote (desktop) eye trackers show that although some non- standard 3rd order polynomial models could provide better accuracy, high-order polynomials do not necessarily provide better results. Different than remote setups though, where gaze is estimated over a relatively narrow field-of-view surface (e.g. less than 30 × 20 degrees on typical computer displays), head-mounted gaze trackers (HMGT) are often desired to cover a relatively wider field-of-view to make sure that the gaze is detected in the scene image even for extreme eye angles. In this paper we investigate the behavior of the gaze estimation error distribution throughout the image of the scene camera when using polynomial functions. Us- ing simulated scenarios, we describe effects of four different sources of error: interpolation, extrapolation, parallax, and radial distortion. We show that the use of third order polynomials result in more accurate gaze estimates in HMGT, and that the use of wide angle lenses might be beneficial in terms of error reduction.

@article{Mardanbegi:JEMR18, 
author = {Diako Mardanbegi and Andrew Kurauchi and Carlos Morimoto}, 
title = {An investigation of the distribution of gaze estimation errors in head mounted gaze trackers using polynomial functions}, 
journal = {Journal of Eye Movement Research}, 
volume = {11}, 
number = {3}, 
year = {2018}, 
keywords = {eye tracking, gaze estimation, Head-mounted eye tracking, polynomial map- ping, error distribution}, 
issn = {1995-8692}, 
url = {https://bop.unibe.ch/JEMR/article/view/4183},
}

Recently, interaction techniques in which the user selects screen targets by matching their movement with the input device have been gaining popularity, particularly in the context of gaze interaction (e.g. Pursuits, Orbits, AmbiGaze, etc.). However, though many algorithms for enabling such interaction techniques have been proposed, we still lack an understanding of how they compare to each other. In this paper, we introduce two new algorithms for matching eye movements: Profile Matching and 2D Correlation, and present a systematic comparison of these algorithms with two other state-of-the-art algorithms: the Basic Correlation algorithm used in Pursuits and the Rotated Correlation algorithm used in PathSync. We also examine the effects of two thresholding techniques and post-hoc filtering. We evaluated the algorithms on a user dataset and found the 2D Correlation with one-level thresholding and post-hoc filtering to be the best performing algorithm.

@inproceedings{Velloso:2018:COD:3204493.3204524, 
author = {Velloso, Eduardo and Coutinho, Flavio Luiz and Kurauchi, Andrew and Morimoto, Carlos H}, 
title = {Circular Orbits Detection for Gaze Interaction Using 2D Correlation and Profile Matching Algorithms}, 
booktitle = {Proceedings of the 2018 ACM Symposium on Eye Tracking Research & Applications -- Paper of Note Award}, 
series = {ETRA '18}, 
year = {2018}, 
isbn = {978-1-4503-5706-7}, 
location = {Warsaw, Poland}, 
pages = {25:1--25:9}, 
articleno = {25}, 
numpages = {9}, 
url = {http://doi.acm.org/10.1145/3204493.3204524}, 
doi = {10.1145/3204493.3204524}, 
acmid = {3204524}, 
publisher = {ACM}, 
address = {New York, NY, USA}, 
keywords = {eye tracking, gaze interaction, orbits, pursuits, smooth pursuits}, 
}

Most applications involving gaze-based interaction are supported by estimation techniques that find a mapping between gaze data and corresponding targets on a 2D surface. However, in Virtual and Augmented Reality (AR) environments, interaction occurs mostly in a volumetric space, which poses a challenge to such techniques. Accurate point-of-regard (PoR) estimation, in particular, is of great importance to AR applications, since most known setups are prone to parallax error and target ambiguity. In this work, we expose the limitations of widely used techniques for PoR estimation in 3D and propose a new calibration procedure using an uncalibrated head-mounted binocular eye tracker coupled with an RGB-D camera to track 3D gaze within the scene volume. We conducted a study to evaluate our setup with real-world data using a geometric and an appearance-based method. Our results show that accurate estimation in this setting still is a challenge, though some gaze-based interaction techniques in 3D should be possible.

@inproceedings{Elmadjian:2018:GES:3206343.3206351,
author = {Elmadjian, Carlos and Shukla, Pushkar and Tula, Antonio Diaz and Morimoto, Carlos H.}, 
title = {3D Gaze Estimation in the Scene Volume with a Head-mounted Eye Tracker}, 
booktitle = {Proceedings of the Workshop on Communication by Gaze Interaction}, 
series = {COGAIN '18}, 
year = {2018}, 
isbn = {978-1-4503-5790-6}, 
location = {Warsaw, Poland}, 
pages = {3:1--3:9}, 
articleno = {3}, 
numpages = {9}, 
url = {http://doi.acm.org/10.1145/3206343.3206351}, 
doi = {10.1145/3206343.3206351}, 
acmid = {3206351}, 
publisher = {ACM}, 
address = {New York, NY, USA}, 
keywords = {3D dataset, calibration, gaze estimation, head-mounted eye tracking}, 
}

Text entry by gazing on a virtual keyboard (also known as eye typing) is an important component of any gaze communication system. One of the main challenges for efficient communication is how to avoid unintended key selections due to the Midas' touch problem. The most common selection technique by gaze is dwelling. Though easy to learn, long dwell-times slows down the communication, and short dwells are prone to error. Context switching (CS{}) is a faster and more comfortable alternative, but the duplication of contexts takes a lot of screen space. In this paper we introduce two new CS{} designs using dynamic expanding targets that are more appropriate when a reduced interaction window is required. We compare the performance of the two new designs with the original CS{} design using QWERTY layouts as contexts. Our results with 6 participants typing with the 3 keyboards show that the use of smaller size layouts with dynamic expanding targets are as accurate and comfortable as the larger QWERTY layout, though providing lower typing speeds.

@inproceedings{Morimoto:2018:CSE:3206343.3206347,
author = {Morimoto, Carlos H. and Leyva, Jose A. T. and Diaz-Tula, Antonio}, 
title = {Context Switching Eye Typing Using Dynamic Expanding Targets}, 
booktitle = {Proceedings of the Workshop on Communication by Gaze Interaction}, 
series = {COGAIN '18}, 
year = {2018}, 
isbn = {978-1-4503-5790-6}, 
location = {Warsaw, Poland}, 
pages = {6:1--6:9}, 
articleno = {6}, 
numpages = {9}, 
url = {http://doi.acm.org/10.1145/3206343.3206347}, 
doi = {10.1145/3206343.3206347}, 
acmid = {3206347}, 
publisher = {ACM}, 
address = {New York, NY, USA}, 
keywords = {context switching, expanding targets, eye tracking, eyetyping, gaze interaction}, 
}

Structured lighting is a computer vision technique that projects illumination patterns onto the scene to facilitate feature extraction from the captured images. The use of low- cost cameras is avoided not only due to their low image quality but mostly due to the lack of a synchronization mechanism for the illuminators. In this paper we propose a method to synchronize low-cost cameras and illuminators based on the dynamic estimation of the camera sensor exposure and number of lines. At the same time, the use of structured stroboscopic lighting is used to enhance the image quality. Starting with a coarse estimation of the sensor parameters, we developed computer vision algorithms that detect image artifacts created by the structured lighting when the illuminators are not correctly synchronized with the camera frames. The detected artifacts are used to refine the estimation of the sensor parameters and to adjust the firing of the illuminators until a clear picture is obtained. Our technique requires a simple external circuit to control the firing of the illuminators, that is adjusted by software, and allows virtually any modern digital camera to be used in structured lighting applications. We demonstrate the use of this technique in a fast 187 fps robust pupil detector that can be used for gaze interaction applications.

@inproceedings{Borsato:17, 
author = {Borsato, Frank H. and Morimoto, Carlos H.}, 
title = {Building Structured Lighting Applications Using Low-Cost Cameras}, 
year = {2017}, 
month = {october}, 
booktitle = {Electronic Proceedings of the 30th Conference on Graphics, Patterns and Images (SIBGRAPI'17)}, 
editor = {M. Lage and L. A. F. Fernandes and R. Marroquim and H. Lopes}, 
address = {Niterói, RJ, Brazil}, 
url = {http://sibgrapi2017.ic.uff.br/} 
}

With eye trackers gradually becoming personal wearable devices, gaze-based interaction will become a relevant technique for wearable applications. However, it is a common belief that high-resolution images and high frame rates are desirable to achieve the accuracy and precision required for human interaction. Because of the high computational load, a wearable eye tracker would have their batteries quickly drained out. In this paper we investigate how much processing power can be saved by lowering these requirements, and still maintain the performance adequate for human interaction. We have conducted an experiment using a head-mounted Pupil Labs eye tracker. Our results from 10 participants show that accuracy and precision remain below one degree of error for image resolution of 240 lines, and frame rates as low as 5 frames per second (FPS). Using this minimum setup, we estimate that power consumption can be reduced by 90% compared to the eye tracker camera regular settings (480 lines and 30 FPS). We also propose an algorithm that successfully detects reading behavior in real-time at 5 FPS in order to demonstrate the usefulness of gaze data at such low rates.

@inproceedings{Elmadjian:17, 
author = {Elmadjian, C.E.L. and Diaz-Tula, A. and Aluani, F.O. and Morimoto, C.H.}, 
title = {Gaze Interaction Using Low Resolution Images at 5 FPS}, 
year = {2017}, 
month = {August}, 
booktitle = {Electronic Proceedings of the COGAIN 2017 Symposium (COGAIN'17)}, 
editor = {Bulling, A. and Bednarik, R. and Hansen, J.P. and Morimoto, C.H.}, 
address = {Wuppertal, Germany}, 
url = {http://cogain2017.cogain.org/?schedule} 
}

Fixations and saccades are commonly used in gaze-based interfaces. State-of-the-art algorithms for eye movement segmentation work well for high speed and accurate eye trackers, which are still too expensive and bulky for most gaze interaction applications. For low-end eye trackers running at 30 to 60 Hz and with accuracy of about 1 degree, such algorithms do not perform as well. We propose a robust, real-time method to classify eye movement data into four categories: fixations, saccades, drifts, and none. The classifier is based on a finite-state machine (FSM) and is robust to missing data and blinks. The approach first filters raw gaze data to recover missing samples and smoothes the data. The current filtered sample is then classified by computing spatial dispersion and absolute eye velocity using a small number of recent gaze samples and the current state of the machine. Qualitative evaluation have shown evidence that FSM reduces latency after blinks, reduces the number of refocusing events and improves user experience during the interaction compared with a simple fixation detector based on a running average window. The source code is publicly available at https://bitbucket.org/diaztula/gaze_movements_fsm/.

@inproceedings{Diaz-Tula:17, 
author = {Diaz-Tula, A. and Morimoto, C.H.}, 
title = {Robust, real-time eye movement classification for gaze interaction using finite state machines}, 
year = {2017}, 
month = {August}, 
booktitle = {Electronic Proceedings of the COGAIN 2017 Symposium (COGAIN'17)}, 
editor = {Bulling, A. and Bednarik, R. and Hansen, J.P. and Morimoto, C.H.}, 
address = {Wuppertal, Germany}, 
url = {http://cogain2017.cogain.org/?schedule} 
}

Eye gaze is an important source of information to evaluate computer interfaces. Typically, visualization of gaze data is performed using heatmaps and gaze scanpaths displayed on top of images of the interface, enhancing regions that have attracted the user’s visual attention. Such tools work well for static interfaces but they are not appropriate to visualize dynamic interfaces where the object of interaction is always changing, such as games, web browsing, or even common applications that change the interface according to the status of the application. In this paper we introduce an interactive tool to explore the spatial-temporal distribution of visual attention called Heatmap Explorer (HME). HME allows the experimenter to control the visualization by selecting temporal intervals and adjusting filter parameters of the eye movement classification algorithm. We show results of three typical application scenarios and discuss how HME can be an effective usability evaluation tool.

@inproceedings{DiazTual:IHC2016,
title = {Heatmap Explorer: an interactive gaze data visualization tool for the evaluation of computer interfaces},
author = {Antonio Diaz Tula, Andrew T. N. Kurauchi, Flávio L. Coutinho, and Carlos H. Morimoto},
booktitle = {Proc. of the 15th Brazilian Symposium on Human Factors in Computing Systems (IHC'16)},
series = {IHC'16},
year = {2016},
month = {October},
address = {São Paulo,Brazil}
}

Eye-typing is an important tool for people with physical disabilities and, for some, it is their main form of communication. By observing expert typists using physical keyboards, we notice that visual throughput is considerably reduced in current eye-typing solutions. We propose AugKey to improve throughput by augmenting keys with a prefix, to allow continuous text inspection, and suffixes to speed up typing with word prediction. AugKey limits the visual information to the foveal region to minimize eye movements (i.e., reduce eye work). We have applied AugKey to a dwell-time keyboard and compared its performance with two conditions with no augmented feedback: a keyboard with and one without word prediction. Results show that AugKey can be about 28% faster than no word prediction and 20% faster than traditional word prediction, with a smaller workload index.

@inproceedings{Diaz-Tula:CHI2016,
 author = {Antonio Diaz Tula and Carlos H. Morimoto},
 title = {AugKey: Increasing Foveal Throughput in Eye Typing with Augmented Keys},
 booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
 series = {CHI '16},
 year = {2016},
 isbn = {978-1-4503-3362-7},
 location = {Santa Clara, California, USA},
 pages = {3533--3544},
 numpages = {12},
 url = {http://doi.acm.org/10.1145/2858036.2858517},
 doi = {10.1145/2858036.2858517},
 acmid = {2858517},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {augmented feedback, eye typing, text-entry speed, user experience, word prediction},
}

Text entry using gaze-based interaction is a vital communication tool for people with motor impairments. Most solutions require the user to fixate on a key for a given dwell time to select it, thus limiting the typing speed. In this paper we introduce EyeSwipe, a dwell-time-free gaze-typing method. With EyeSwipe, the user gaze-types the first and last characters of a word using the novel selection mechanism "reverse crossing." To gaze-type the characters in the middle of the word, the user only needs to glance at the vicinity of the respective keys. We compared the performance of EyeSwipe with that of a dwell-time-based virtual keyboard. EyeSwipe afforded statistically significantly higher typing rates and more comfortable interaction in experiments with ten participants who reached 11.7 words per minute (wpm) after 30 min typing with EyeSwipe.

@inproceedings{Kurauchi:CHI2016,
 author = {Andrew T. N. Kurauchi and Wenxin Feng and Ajjen Joshi and Carlos H. Morimoto and Margrit Betke},
 title = {EyeSwipe: Dwell-free Text Entry Using Gaze Paths},
 booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
 series = {CHI '16},
 year = {2016},
 isbn = {978-1-4503-3362-7},
 location = {Santa Clara, California, USA},
 pages = {1952--1956},
 numpages = {5},
 url = {http://doi.acm.org/10.1145/2858036.2858335},
 doi = {10.1145/2858036.2858335},
 acmid = {2858335},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {dwell-free typing, eye tracking, eye typing, target selection, text entry},
}

Video-based eye trackers (VETs) have become the dominant eye tracking technology due to its reasonable cost, accuracy, and easy of use. VETs require real-time image processing to detect and track eye features such as the center of the pupil and corneal reflection to estimate the point of regard. Despite the continuous evolution of cameras and computers that made head mounted eye trackers easier to use in natural activities, real-time processing of high resolution images in mobile devices remains a challenge. In this paper we investigate the feasibility of a novel eye-tracking technique intended for wearable applications that use mice chips as imaging sensors. Such devices are widely available at very low cost, and provide high speed and accurate 2D tracking data. Though mice chips have been used for many purposes other than a computer's pointing device, to our knowledge this is the first attempt to use it as an eye tracker. To validate the technique, we built an episcleral database with about 100 high resolution episcleral patches from 7 individuals. The episclera is the outer most layer of the sclera, which is the white part of the eye, and consists of dense vascular connective tissue. We have used the patches to determine if the episclera contains enough texture to be reliably tracked. We also present results from a prototype built using an off-the-shelf mouse sensor. Our results show that a mouse-based eye tracker has the potential to be very accurate, precise, and fast (measuring 2.1' of visual angle at 1 KHz speed), with little overhead for the wearable computer.

@inproceedings{Borsato:ETRA2016,
 author = {Frank H. Borsato and Carlos H. Morimoto},
 title = {Episcleral Surface Tracking: Challenges and Possibilities for Using Mice Sensors for Wearable Eye Tracking},
 booktitle = {Proceedings of the Ninth Biennial ACM Symposium on Eye Tracking Research & Applications},
 series = {ETRA '16},
 year = {2016},
 isbn = {978-1-4503-4125-7},
 location = {Charleston, South Carolina},
 pages = {39--46},
 numpages = {8},
 url = {http://doi.acm.org/10.1145/2857491.2857496},
 doi = {10.1145/2857491.2857496},
 acmid = {2857496},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {eye tracking, mouse sensor, sclera database, translation sensor},
}

The use of video-based eye trackers (VETs) for gaze interaction have allowed people with severe motor disabilities to communicate more effectively. Nonetheless, current low cost VETs present limited accuracy and precision that impose several design constraints for gaze-based computer applications. In this paper we present an extension of the differential lighting (DL) technique for pupil detection and tracking using active light sources. The original technique was developed for analog interlaced cameras with external sync signal to synchronize the light sources with the video signal. In this paper we introduce the Stroboscopic DL technique that can be used with any rolling shutter camera, even those with no external sync. We have developed a computer vision technique to adjust the firing of the stroboscopic lights. Our new algorithm also exploits characteristics of pupil images to improve the accuracy of the tracking algorithm. Another advantage of the method is that using flashed pulses of light creates a virtual exposure time, reducing motion blur and temporal shear in the video volume. A real-time 187 fps prototype of the system was implemented using a low cost PS3 camera. Experimental results comparing the performance of our algorithm with Starburst show significant accuracy and speed improvement.

@inproceedings{Borsato:2015, 
 author = {Frank H. Borsato and Fernando O. Aluani and Carlos H. Morimoto}, 
 booktitle = {2015 IEEE International Conference on Computer Vision Workshop (ICCVW)}, 
 title = {A Fast and Accurate Eye Tracker Using Stroboscopic Differential Lighting}, 
 year = {2015}, 
 pages = {502-510}, 
 keywords = {cameras;gaze tracking;image restoration;motion estimation;synchronisation;video signal processing;PS3 camera;VET;active light source synchronization;analog interlaced cameras;computer vision technique;external sync signal;flashed light pulses;gaze interaction;gaze-based computer applications;motion blur reduction;motor disabled people communication;pupil detection;pupil tracking;rolling shutter camera;stroboscopic DL technique;stroboscopic differential lighting;temporal shear reduction;video signal;video volume;video-based eye trackers;virtual exposure time;Cameras;Clocks;Estimation;Light sources;Lighting;Synchronization;Tracking}, 
 doi = {10.1109/ICCVW.2015.72}, 
 month = {December},
}

The gap effect refers to a reduction in saccadic reaction time (SRT) to an eccentric target, when the fixation point is removed before the target onset. Though it is known that the gap effect peaks when the fixation point is offset about 200 ms before the onset of the eccentric target, it is unknown how this effect is modulated by stimulus variations. In this paper, we propose and investigate a model of saccadic reaction time as a function of the fixation point brightness gain. The brightness gain is defined as the ratio of the final and initial intensities of the stimulus. We have conducted a typical gap effect experiment with 15 participants, where the brightness of the fixation point was manipulated under four conditions and two gap intervals, at the same time and 200 ms before the onset of the eccentric target. The conditions included removing the fixation point (offset), leaving it with constant brightness (overlap), reducing, and increasing its brightness (lower and higher brightness conditions). Experimental data showed a significant gap effect in the offset and lower brightness conditions when compared to the overlap condition. On the other hand, the SRT was significantly longer for the higher brightness condition than the SRT for the overlap condition. Linear regression analysis using ten values of brightness gain shows that our model fits the data well for the 0- and 200-ms gap, with a coefficient of determination of .89 and .94, respectively.

@article{Diaz-Tula:JAPP2015,
 author = {Antonio Diaz Tula and Carlos H. Morimoto and Ronald D. Ranvaud},
 title = {A mathematical model of saccadic reaction time as a function of the fixation point brightness gain},
 journal = {Attention, Perception, & Psychophysics},
 year = {2015},
 volume = {77},
 number = {6},
 pages = {2153--2165},
 issn = {1943-393X},
 doi = {10.3758/s13414-015-0902-9},
 url = {http://dx.doi.org/10.3758/s13414-015-0902-9}
}

Augmentative and alternative communication tools allow people with severe motor disabilities to interact with computers. Two commonly used tools are video-based interfaces and eye trackers. Video-based interfaces map head movements captured by a camera to mouse pointer movements. Alternatively, eye trackers place the mouse pointer at the estimated position of the user's gaze. Eye tracking based interfaces have been shown to even outperform traditional mice in terms of speed, however the accuracy of current eye trackers is not enough for fine mouse pointer placement. In this paper we propose the Head Movement And Gaze Input Cascaded (HMAGIC) pointing technique that combines head movement and gaze-based inputs in a fast and accurate mouse-replacement interface. The interface initially places the pointer at the estimated gaze position and then the user makes fine adjustments with their head movements. We conducted a user experiment to compare HMAGIC with a mouse-replacement interface that uses only head movements to control the pointer. Experimental results indicate that HMAGIC is significantly faster than the head-only interface while still providing accurate mouse pointer positioning.

@inproceedings{Kurauchi:2015,
 author = {Andrew T. N. Kurauchi and Wenxin Feng and Carlos H. Morimoto and Margrit Betke},
 title = {HMAGIC: Head Movement and Gaze Input Cascaded Pointing},
 booktitle = {Proceedings of the 8th ACM International Conference on PErvasive Technologies Related to Assistive Environments},
 series = {PETRA '15},
 year = {2015},
 isbn = {978-1-4503-3452-5},
 location = {Corfu, Greece},
 pages = {47:1--47:4},
 articleno = {47},
 numpages = {4},
 url = {http://doi.acm.org/10.1145/2769493.2769550},
 doi = {10.1145/2769493.2769550},
 acmid = {2769550},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {assistive technology, gaze-based interaction, video-based mouse-replacement interface},
}

This paper introduces the 5* Magic Wand: a user interface that provides an inexpensive way for input of 3D data. Drawing upon the metaphor of the "wizard", a user can point to places, perform gestures and cast "spells" to interact with the computer. We describe a computer vision technique that tracks the wand with 5 degrees of freedom. A prototype implementation, composed of a standard PC, a RGBD camera and a hand-made stick, is presented. Finally, a demo application showing how the wand may be used to navigate on planet Earth is presented.

@inproceedings{Sousa:SVR2015, 
 author = {Alexandre M. F. de Sousa and Carlos H. Morimoto}, 
 booktitle = {Virtual and Augmented Reality (SVR), 2015 XVII Symposium on}, 
 title = {5* Magic Wand: An RGBD Camera-Based 5 DoF User Interface for 3D Interaction}, 
 year = {2015}, 
 pages = {15-22}, 
 keywords = {computer vision;human computer interaction;user interfaces;3D data;3D interaction;5* magic wand;RGBD camera-based 5DoF user interface;computer vision technique;hand-made stick;prototype implementation;Cameras;Computer vision;Computers;Navigation;Switches;Three-dimensional displays;User interfaces;3D user interface;Kinect;RGBD sensor;magic wand;virtual reality}, 
 doi = {10.1109/SVR.2015.10}, 
 month = {May}
}

This paper introduces the 5* Magic Wand, an inexpensive pointing device for 3D interaction. As users play the role of a “wizard”, the 5* Magic Wand allows them to interact with the computer by pointing to places, performing gestures, and casting “spells”. We have developed computer vision techniques to track the wand with 5 degrees of freedom, and built a real-time prototype consisting of a standard PC, a RGBD camera, and a simple stick. Finally, in order to demonstrate the applicability of the system for navigating in virtual environments, we have created a 3D application called “Magic Carpet”.

@article{Sousa:JIS2015,
 title = {5* Magic Wand: a RGBD camera-based 5 DoF pointing device for 3D interaction},
 author = {Alexandre M. F. de Sousa and Carlos H. Morimoto},
 journal = {SBC Journal on Interactive Systems},
 volume = {6},
 number = {2},
 pages = {22-29},
 year = {2015}
}

In this paper we investigate the performance of a gaze-based interaction system that combines Dynamic Context Switching and Meta-Context Switching. These methods are extensions of the Context Switching interaction paradigm. The original context switching idea uses fixed-size contexts. Each context carries the same information, so the user can browse freely within a context without worrying about the Midas touch problem. A saccade to the other context triggers the selection of the item under focus. Dynamic context switching dynamically adjusts the size of a context to improve its useful area, where the context that has the user focus is displayed in full size and the other is minimized. Meta-context switching uses meta-keys to allow the user to escape from the current task and select other contexts or change the operation mode. We have designed and conducted two user experiments to evaluate these new gaze interaction techniques and compare them with selection by dwell time in a search task. The task required browsing through several pages using meta-keys. The experimental results show that dynamic context switching improves user performance when compared to fixed-size context switching and do not cause disorientation. The error rate was significantly higher for dwell time due to the Midas touch problem, although the time spent to complete the task was similar for dwell time and dynamic context switching.

@article{Diaz-Tula:JIS2015,
 title = {Dynamic and Meta-Context Switching for Gaze-Based Interaction},
 author = {Antonio Diaz Tula and Carlos H. Morimoto},
 journal = {SBC Journal on Interactive Systems},
 volume = {6},
 number = {1},
 year = {2015},
 pages = {66-75}
}

Tangible User Interfaces (TUIs) allow interaction with digital content through physical artifacts, enabling a smooth bridge between the digital and the physical worlds. In an entertainment context, such interfaces offer rich expressive capabilities for experimental works. This paper introduces Tangible Beats: a user interface that, in a collaborative fashion, enables the creation of musical beats by manipulating colored tokens on a flat surface. A prototype implementation, composed of a Kinect camera, a video projector, speakers and a laptop is presented. The Kinect is used to estimate the position of the tokens and the projector provides visual feedback to help inexpert music makers compose their own beats by changing the notes and the instruments.

@inproceedings{Sousa:2014, 
 author = {Alexandre M. F. de Sousa and Carlos H. Morimoto}, 
 booktitle = {XIII Simpósio Brasileiro de Jogos e Entretenimento Digital (SBGames) - Trilha de Arte & Design}, 
 title = {Tangible Beats: a collaborative user interface for creating musical beats}, 
 year = {2014}, 
 pages = {304-307}, 
 keywords = {tangible user interfaces, interactive art, collaborative music interface, exploratory design}, 
 month = {November},
}

Realistic makeup simulation can be used to create interactive virtual environments which users can try makeup products in a fast, low-cost and flexible way. This paper presents an interactive virtual mirror for makeup simulation and we propose a interaction model based on a person applying makeup using a mirror. A touchscreen monitor mimics the mirror and the user can apply makeup touching the screen in any face location creating different makeup styles. Once the makeup has been applied, the system is able to keep its position during face movement in real time and without the need of markers, expensive equipment or an initial step to create a user model.

@inproceedings{Campos:IHC2014,
 author = {Filipe M. S. de Campos and Carlos H. Morimoto},
 title = {Espelho Virtual Interativo Para Simulação De Maquiagem},
 booktitle = {Proceedings of the 13th Brazilian Symposium on Human Factors in Computing Systems},
 series = {IHC '14},
 year = {2014},
 isbn = {978-85-7669-291-1},
 location = {Foz do Iguaçu, Brazil},
 pages = {345--348},
 numpages = {4},
 url = {http://dl.acm.org/citation.cfm?id=2738055.2738113},
 acmid = {2738113},
 publisher = {Sociedade Brasileira de Computação},
 address = {Porto Alegre,  Brazil, Brazil},
 keywords = {human computer interaction, makeup simulation},
}

This paper describes the development of Tangible Beats, a tangible musical interface for the creation and collaborative exploration of musical patterns. Tangible Beats allows users to collaboratively play musical patterns by manipulating colored tokens on a flat grid surface. Inexperienced players can explore and express their musicality. Our prototype uses a Kinect camera to detect the position of the colored objects. Visual feedback is provided using a video projector.

@inproceedings{Sousa:IHC2014,
 author = {Alexandre M. F. de Sousa and Carlos H. Morimoto},
 title = {Tangible Beats: Uma Interface Tangível Para Expressão e Exploração Musical},
 booktitle = {Proceedings of the 13th Brazilian Symposium on Human Factors in Computing Systems},
 series = {IHC '14},
 year = {2014},
 isbn = {978-85-7669-291-1},
 location = {Foz do Iguaçu, Brazil},
 pages = {385--388},
 numpages = {4},
 url = {http://dl.acm.org/citation.cfm?id=2738055.2738128},
 acmid = {2738128},
 publisher = {Sociedade Brasileira de Computação},
 address = {Porto Alegre,  Brazil, Brazil},
 keywords = {Kinect, collaborative music expression, exploration, tangible interface},
}

This paper introduces the concept of meta-keys with the objective to extend the currently limited functionality of gaze-based interaction. Meta-keys are two-step gaze gestures between the interface and external markers. A bridge between the interface and the markers avoids accidental activation and reduces the effect of the eye tracking noise. Results of a user study showed that meta-keys have the potential to extend the functionality of gaze-based interaction with a low error rate.

@inproceedings{Diaz-Tula:IHC2014,
 author = {Antonio Diaz Tula and Carlos H. Morimoto},
 title = {Meta-keys: Extending the Functionality of Gaze-based Interaction},
 booktitle = {Proceedings of the 13th Brazilian Symposium on Human Factors in Computing Systems},
 series = {IHC '14},
 year = {2014},
 isbn = {978-85-7669-291-1},
 location = {Foz do Iguaçu, Brazil},
 pages = {285--292},
 numpages = {8},
 url = {http://dl.acm.org/citation.cfm?id=2738055.2738101},
 acmid = {2738101},
 publisher = {Sociedade Brasileira de Computação},
 address = {Porto Alegre,  Brazil, Brazil},
 keywords = {gaze gesture, gaze-based interaction, meta-keys},
}

Gaze interaction has been helping people with physical disabilities for many years but its use is still limited, in part because it has been designed for desktop platforms. In this paper we discuss how gaze-aware wearable computing can be used to create new applications that can significantly enhance the quality of life of people with disabilities, including the elderly, and also be used to promote digital inclusion for illiterate people.

@inproceedings{Kurauchi:IHC2014,
 author = {Andrew T. N. Kurauchi and Antonio Diaz Tula and Carlos H. Morimoto},
 title = {Facilitating Accessibility and Digital Inclusion Using Gaze-aware Wearable Computing},
 booktitle = {Proceedings of the 13th Brazilian Symposium on Human Factors in Computing Systems},
 series = {IHC '14},
 year = {2014},
 isbn = {978-85-7669-291-1},
 location = {Foz do Iguaçu, Brazil},
 pages = {417--420},
 numpages = {4},
 url = {http://dl.acm.org/citation.cfm?id=2738055.2738140},
 acmid = {2738140},
 publisher = {Sociedade Brasileira de Computação},
 address = {Porto Alegre, Brazil, Brazil},
 keywords = {accessibility, augmented cognition, digital inclusion, gaze interaction, gaze tracking, wearable computing},
}

As técnicas de estimação do olhar em rastreadores de olhar móveis assumem que o usuário está sempre olhando para um mesmo plano. Essa suposição é aceitável quando o olhar do usuário fica limitado a um monitor de computador, mas introduz um erro quando o rastreador é utilizado para capturar o olhar sobre objetos 3D em ambientes naturais. Um componente significativo desse erro é causado pela paralaxe entre o olho e a câmera da cena do rastreador móvel. Neste trabalho apresentamos um novo método que explora propriedades invariantes da geometria projetiva para a compensação desse erro de paralaxe com o uso de dois planos de calibração. Por meio de simulações, mostramos que o método é viável e promissor, sendo bem mais robusto a variações de profundidade do que os métodos tradicionais, apresentando erros médios até 2.5 vezes menores.

@inproceedings{Kurauchi:CBEB2014,
 title = {Método de calibração de rastreadores de olhar móveis utilizando 2 planos para correção de paralaxe},
 author = {Andrew T. N. Kurauchi and Antonio Diaz Tula and Carlos H. Morimoto},
 booktitle = {XXIV Congresso Brasileiro de Engenharia Biomédica},
 series = {CBEB '14},
 pages = {2683--2686},
 year = {2014}
}

O estudo dos movimentos oculares é muito importante na identificação de disfunções oculomotoras. No entanto, os sistemas comerciais de rastreamento do olhar possuem custos muito elevados e mobilidade limitada. Nesse artigo apresentamos um sistema de rastreamento do olhar móvel de baixo custo montado na cabeça, voltado ao estudo dos movimentos oculares para a identificação de disfunções oculomotoras. São apresentadas as descrições de hardware e software desenvolvidos, além de um exemplo prático do seu uso em teste de acompanhamento visual. O sistema proposto pode ajudar na avaliação oculomotora dos pacientes, buscando melhorar sua qualidade de vida.

@inproceedings{Diaz-Tula:CBEB2014,
 title = {Sistema móvel de baixo custo para rastreamento do olhar voltado à identificação de disfunções oculomotoras},
 author = {Antonio Diaz Tula and Andrew T. N. Kurauchi and Carlos H. Morimoto and Silvia Veitzman and Jéssica N. Ianof},
 booktitle = {XXIV Congresso Brasileiro de Engenharia Biomédica},
 series = {CBEB '14},
 pages = {2830--2833},
 year = {2014}
}

Realistic makeup simulation can be used to create interactive virtual environments where users can try makeup products in a fast, low-cost and flexible way. Most existing makeup simulators are not appropriate for interactive applications because they don't allow the realistic application of specific products interactively or are based on fixed makeup styles that depend on the same pose and lighting conditions of predefined makeup templates. This paper introduces a new method for makeup simulation that receives a picture with a clean (no makeup) face, and decomposes the facial region into several frequency bands. Each band is processed individually to conceal or enhance facial texture and color features that can be affected by makeup. A specific makeup product is defined by a set of properties that change the texture and color features. To allow the interaction among several makeup products, the method uses a layered representation that can mimic the application of a sequence of products. We present a real-time implementation of the method that simulates the application of foundation, eye shadow and lipstick. The software was developed in C++ and can process one high resolution (1018×990) facial image in about 80 ms and one low resolution (640×480) in about 30ms.

@inproceedings{Campos:SVR2014, 
 author = {Filipe M. S. de Campos and Carlos H. Morimoto}, 
 booktitle = {XVI Symposium on Virtual and Augmented Reality (SVR)}, 
 title = {Virtual Makeup: Foundation, Eye Shadow and Lipstick Simulation}, 
 year = {2014}, 
 pages = {181-189}, 
 keywords = {cosmetics;face recognition;image colour analysis;image texture;virtual reality;C++;color feature;eye shadow;facial region;facial texture;foundation;interactive virtual environment;lighting condition;lipstick simulation;realistic makeup simulation;virtual makeup;Computational modeling;Face;Image color analysis;Image resolution;Irrigation;Solid modeling;Three-dimensional displays;makeup simulation; image processing.}, 
 doi = {10.1109/SVR.2014.32}, 
 month = {May}
}

Current WIMP (Windows, Icons, Menus, Pointers) interfaces used in most computer game map editors impose complex interactions to level designers. In this paper we present a novel map editing tool based on a Natural User Interface (NUI) that uses common drawing objects, such as pens and erasers, to build and edit map elements, as well as hand gestures to manipulate them. As a proof-of-concept, we have implemented a tile-based map editor featuring a 3D NUI with multitouch capabilities. The real-time prototype, built using two Microsoft Kinect devices and a regular LCD screen, displays the levels in 3D while adjusting the perspective according to the location of the user.

@inproceedings{Sousa:SBGames2013,
  title = {Penspective: a Perspective-Aware Game Map Editor with Natural Interaction},
  author = {Alexandre M. F. de Sousa and Anderson C. M. Tavares and Carlos H. Morimoto},
  booktitle = {XII Simpósio Brasileiro de Jogos e Entretenimento Digital},
  series = {SBGames '13},
  year = {2013},
  month = {October},
  address = {São Paulo,Brazil}
}

Augmented cognition applications must deal with the problem of how to exhibit information in an orderly, understandable, and timely fashion. Though context have been suggested to control the kind, amount, and timing of the information delivered, we argue that gaze can be a fundamental tool to reduce the amount of information and provide an appropriate mechanism for low and divided attention interaction. We claim that most current gaze interaction paradigms are not appropriate for wearable computing because they are not designed for divided attention. We have used principles suggested by the wearable computing community to develop a gaze supported augmented cognition application with three interaction modes. The application provides information of the person being looked at. The continuous mode updates information every time the user looks at a different face. The key activated discrete mode and the head gesture activated mode only update the information when the key is pressed or the gesture is performed. A prototype of the system is currently under development and it will be used to further investigate these claims.

@inproceedings{Kurauchi:CHI2013, 
  author = {Andrew T. N. Kurauchi and Carlos H. Morimoto and Diako Mardanbeigi and Dan W. Hansen}, 
  title = {Towards Wearable Gaze Supported Augmented Cognition}, 
  booktitle = {CHI 2013 Workshop on Gaze Interaction in the Post-WIMP World}, 
  year = {2013}, 
  month = {April}, 
  address = {Paris, France}, 
}

Many results from psychophysics have indicated that the latency of saccadic eye movements is affected by how new visual stimuli is presented. In this paper we show how two such results, known as the gap effect (GE) and overlap condition (OC), can be used to improve gaze interaction. We have chosen a dwell time based eye typing application, since eye typing can be easily modeled as a sequence of eye movements from one key to the next. By modeling how dwell time selection is performed, we show how the GE and OC can be used to generate visual feedback that facilitates the eye movement to the next key. A pilot experiment was conducted in which participants had to type short phrases on a virtual keyboard using 2 different visual feedback methods, one traditional feedback based on animation and a new feedback scheme using the GE and OC. Results show that using a feedback that exploits these phenomena facilitates eye movements and can improve eye typing user experience and performance.

@inproceedings{Diaz-Tula:CHI2013,
 author = {Antonio Diaz Tula and Andrew T. N. Kurauchi and Carlos H. Morimoto},
 title = {Facilitating Gaze Interaction Using the Gap and Overlap Effects},
 booktitle = {CHI '13 Extended Abstracts on Human Factors in Computing Systems},
 series = {CHI EA '13},
 year = {2013},
 isbn = {978-1-4503-1952-2},
 location = {Paris, France},
 pages = {91--96},
 numpages = {6},
 url = {http://doi.acm.org/10.1145/2468356.2468374},
 doi = {10.1145/2468356.2468374},
 acmid = {2468374},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {eye typing, gap effect, gaze interaction, overlap condition},
}

When first introduced, the cross-ratio (CR) based remote eye tracking method offered many attractive features for natural human gaze-based interaction, such as simple camera setup, no user calibration, and invariance to head motion. However, due to many simplification assumptions, current CR-based methods are still sensitive to head movements. In this paper, we revisit the CR-based method and introduce two new extensions to improve the robustness of the method to head motion. The first method dynamically compensates for scale changes in the corneal reflection pattern, and the second method estimates true coplanar eye features so that the cross-ratio can be applied. We present real-time implementations of both systems, and compare the performance of these new methods using simulations and user experiments. Our results show a significant improvement in robustness to head motion and, for the user experiments in particular, an average reduction of up to 40 % in gaze estimation error was observed.

@article{Coutinho:IJCV2013,
 author = {Flavio L. Coutinho and Carlos H. Morimoto},
 title = {Improving Head Movement Tolerance of Cross-Ratio Based Eye Trackers},
 journal = {International Journal of Computer Vision},
 issue_date = {February  2013},
 volume = {101},
 number = {3},
 month = {feb},
 year = {2013},
 issn = {0920-5691},
 pages = {459--481},
 numpages = {23},
 url = {http://dx.doi.org/10.1007/s11263-012-0541-8},
 doi = {10.1007/s11263-012-0541-8},
 acmid = {2440686},
 publisher = {Kluwer Academic Publishers},
 address = {Hingham, MA, USA},
 keywords = {Cross-ratio, Eye tracking, Free-head motion, Gaze tracking, Head movement tolerance, Homography, Remote eye gaze tracking},
}

This paper introduces Dynamic Context Switching (DCS) as an extension of the Context Switching (CS) paradigm for gaze-based interaction. CS replicates information in each context. The user can freely explore one context without worrying about the Midas touch problem, and a saccade to the other context triggers the selection of the item under focus. Because CS has to display two contexts simultaneously, the amount of useful screen space is limited. DCS dynamically adjusts the context sizes, where the context that has the focus is displayed in full size, while the other is minimized, thus improving useful screen space. A saccade to the minimized context triggers selection, and properly readjusts the sizes of the contexts. Results from a pilot user experiment show that DCS improves user performance and do not cause disorientation due to the dynamic context resizing.

@inproceedings{Diaz-Tula:ETRA2012,
 author = {Antonio Diaz Tula and Filipe M. S. de Campos and Carlos H. Morimoto},
 title = {Dynamic Context Switching for Gaze Based Interaction},
 booktitle = {Proceedings of the Symposium on Eye Tracking Research & Applications},
 series = {ETRA '12},
 year = {2012},
 isbn = {978-1-4503-1221-9},
 location = {Santa Barbara, California},
 pages = {353--356},
 numpages = {4},
 url = {http://doi.acm.org/10.1145/2168556.2168635},
 doi = {10.1145/2168556.2168635},
 acmid = {2168635},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {dynamic context switching, gaze based interaction, selection by gaze},
}

Remote gaze estimation using a single non-calibrated camera, simple user calibration or calibration free, and robust to head movements are very desirable features of eye tracking systems. Because cross-ratio (CR) is an invariant property of projective geometry, gaze estimation methods that rely on this property have the potential to provide these features, though most current implementations rely on a few simplifications that compromise the performance of the method. In this paper, the CR method for gaze tracking is revisited, and we introduce a new method that explicitly compensates head movements using a simple 3 parameter eye model. The method uses a single non-calibrated camera and requires a simple calibration procedure per user to estimate the eye parameters. We have conducted simulations and experiments with real users that show significant improvements over current state-of-the-art CR methods that do not explicitly compensate for head motion.

@inproceedings{Coutinho:ETRA2012,
 author = {Flávio L. Coutinho and Carlos H. Morimoto},
 title = {Augmenting the Robustness of Cross-ratio Gaze Tracking Methods to Head Movement},
 booktitle = {Proceedings of the Symposium on Eye Tracking Research and Applications},
 series = {ETRA '12},
 year = {2012},
 isbn = {978-1-4503-1221-9},
 location = {Santa Barbara, California},
 pages = {59--66},
 numpages = {8},
 url = {http://doi.acm.org/10.1145/2168556.2168565},
 doi = {10.1145/2168556.2168565},
 acmid = {2168565},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {cross-ratio, eye gaze tracking, head movement compensation},
}

Automated virtual camera control has been widely used in animation and interactive virtual environments. We have developed a multiple sparse camera based free view video system prototype that allows users to control the position and orientation of a virtual camera, enabling the observation of a real scene in three dimensions (3D) from any desired viewpoint. Automatic camera control can be activated to follow selected objects by the user. Our method combines a simple geometric model of the scene composed of planes (virtual environment), augmented with visual information from the cameras and pre-computed tracking information of moving targets to generate novel perspective corrected 3D views of the virtual camera and moving objects. To achieve real-time rendering performance, view-dependent textured mapped billboards are used to render the moving objects at their correct locations and foreground masks are used to remove the moving objects from the projected video streams. The current prototype runs on a PC with a common graphics card and can generate virtual 2D views from three cameras of resolution 768x576 with several moving objects at about 11fps.

@article{Silva:CG2011,
 author = {Jeferson R. Silva and Thiago T. Santos and Carlos H. Morimoto},
 title = {Virtual Reality in Brazil: Automatic Camera Control in Virtual Environments Augmented Using Multiple Sparse Videos},
 journal = {Computers and Graphics},
 issue_date = {April, 2011},
 volume = {35},
 number = {2},
 month = {apr},
 year = {2011},
 issn = {0097-8493},
 pages = {412--421},
 numpages = {10},
 url = {http://dx.doi.org/10.1016/j.cag.2011.01.012},
 doi = {10.1016/j.cag.2011.01.012},
 acmid = {1961737},
 publisher = {Pergamon Press, Inc.},
 address = {Elmsford, NY, USA},
 keywords = {Automatic virtual camera control, Free viewpoint video, Multiple sparse cameras, Real-time image and video synthesis},
}

This paper introduces the Laboratory of Technologies for Interaction (LaTIn) and briefly describes its current main projects. The main focus of LaTIn has been developing new ways of human-machine interaction using computer vision techniques. The projects are categorized according to the distance between the human user and the machine being operated. For close distances, appropriate for interaction with desktop computers for example, we have developed eye-gaze based interfaces. For mid range distances, we have built hand and body gestures interfaces that are appropriate for virtual and augmented reality settings and, for large distances, we have developed novel multiple people tracking techniques that have been used for surveillance and monitoring applications.

@article{Morimoto:SBCJIS2011,
 author = {Carlos H. Morimoto and Flavio L. Coutinho and Jefferson R. da Silva and Silvia E. Ghirotti and Thiago T. Santos},
 title = {Computer Vision based Machine Interaction},
 journal = {SBC Journal on 3D Interactive Systems},
 year = {2011},
 volume = {2},
 number = {2},
 issn = {2236-3297},
 pages = {55-58},
 numpages = {4}
}

With the steady price reduction of 3D visualization devices such as Head Mounted Displays (HMDs) and more recently 3D TVs, we can now foresee the dissemination of applications that was only possible within very expensive virtual reality environments. However, interaction within virtual 3D environments requires more natural modes than those provided by the ubiquitous mouse and keyboard. In this paper we introduce a novel low cost 3D hand gesture based interaction system. We have developed a real-time stereo computer vision system and a hybrid interface that combines natural and symbolic gestures for navigation and manipulation of 3D objects in virtual environments. Results from a pilot study reveals that the hybrid interface presents great power and flexibility without significant increase in the complexity of the user interaction.

@inproceedings{Ghirotti:IHC2010,
 author = {Silvia E. Ghirotti and Carlos H. Morimoto},
 title = {Um Sistema De Interação Baseado Em Gestos Manuais Tridimensionais Para Ambientes Virtuais},
 booktitle = {Proceedings of the IX Symposium on Human Factors in Computing Systems},
 series = {IHC '10},
 year = {2010},
 location = {Belo Horizonte, Minas Gerais, Brazil},
 pages = {159--168},
 numpages = {10},
 url = {http://dl.acm.org/citation.cfm?id=1999593.1999611},
 acmid = {1999611},
 publisher = {Brazilian Computer Society},
 address = {Porto Alegre, Brazil, Brazil},
 keywords = {3D gesture recognition, gesture interface, interaction in 3D virtual environments},
}

Free Viewpoint Video (FVV) might become one of the most interesting features in future digital television, enabling viewers to move around freely and observe the scene in three dimensions (3D) from any desired viewpoint, similar to many 3D interactive computer graphics applications today. This paper describes a FVV appropriate for virtual reality systems that allows the user to explore a 3D scene in real time using a small set of sparse cameras. Our method combines a simple geometric model of the scene composed of planes with tracking information of moving targets to generate novel perspective corrected 3D views of a virtual camera. To achieve real-time rendering frame rates, view dependent textured mapped billboards are used to render the moving objects at their correct locations and foreground masks are used to remove the moving objects from the projected video streams. Our current real-time prototype is able to generate an interactive 2D virtual camera view from 3 synchronized video streams of resolution 768×576 with several moving objects at about 12 fps.

@inproceedings{Silva:SVR2010,
 author = {Jeferson R. da Silva and Thiago T. Santos and Carlos H. Morimoto},
 title = {Real Time Novel View Scene Rendering From Multiple Sparse Videos},
 booktitle = {Proceedings of the XII Brazilian Symposium on Virtual and Augmented Reality},
 series = {SVR '10},
 year = {2010},
 publisher = {Sociedade Brasileira de Computação},
 address = {Natal, Brazil},
 pages = {184-193}
}

This paper proposes a method to locate and track people by combining evidence from multiple cameras using the homography constraint. The proposed method use foreground pixels from simple background subtraction to compute evidence of the location of people on a reference ground plane. The algorithm computes the amount of support that basically corresponds to the “foreground mass” above each pixel. Therefore, pixels that correspond to ground points have more support. The support is normalized to compensate for perspective effects and accumulated on the reference plane for all camera views. The detection of people on the reference plane becomes a search for regions of local maxima in the accumulator. Many false positives are filtered by checking the visibility consistency of the detected candidates against all camera views. The remaining candidates are tracked using Kalman filters and appearance models. Experimental results using challenging data from PETS’06 show good performance of the method in the presence of severe occlusion. Ground truth data also confirms the robustness of the method.

@article{Santos:PRL2011,
 title = {Multiple camera people detection and tracking using support integration},
 journal = {Pattern Recognition Letters},
 volume = {32},
 number = {1},
 pages = {47 - 55},
 year = {2011},
 note = {Image Processing, Computer Vision and Pattern Recognition in Latin America},
 issn = {0167-8655},
 doi = {http://dx.doi.org/10.1016/j.patrec.2010.05.016},
 url = {http://www.sciencedirect.com/science/article/pii/S016786551000156X},
 author = {Thiago T. Santos and Carlos H. Morimoto},
 keywords = {People tracking, Multiple view integration, Video surveillance and monitoring, Homography constraint}
}

Traditional cross-ratio methods (TCR) project a light pattern and use invariant properties of projective geometry to estimate the gaze position. Advantages of the TCR methods include robustness to large head movements and in general requires just a one time per user calibration. However, the accuracy of TCR methods decay significantly for head movements along the camera optical axis, mainly due to the angular difference between the optical and visual axis of the eye. In this paper we propose a depth compensation cross-ratio (DCR) method that improves the accuracy of TCR methods for large head depth variations. Our solution compensates the angular offset using a 2D onscreen vector computed from a simple calibration procedure. The length of the 2D vector, which varies with head distance, is adjusted by a scale factor that is estimated from relative size variations of the corneal reflection pattern. The proposed DCR solution was compared to a TCR method using synthetic and real data from 2 users. An average improvement of 40% was observed with synthetic data, and 8% with the real data.

@inproceedings{Coutinho:ETRA2010,
 author = {Flavio L. Coutinho and Carlos H. Morimoto},
 title = {A Depth Compensation Method for Cross-ratio Based Eye Tracking},
 booktitle = {Proceedings of the 2010 Symposium on Eye-Tracking Research & Applications},
 series = {ETRA '10},
 year = {2010},
 isbn = {978-1-60558-994-7},
 location = {Austin, Texas},
 pages = {137--140},
 numpages = {4},
 url = {http://doi.acm.org/10.1145/1743666.1743700},
 doi = {10.1145/1743666.1743700},
 acmid = {1743700},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {depth compensation, depth estimation, free head motion, single camera eye gaze tracking},
}

This paper presents context switching as an alternative to selection by dwell time. The technique trades screen space for comfort and speed. By replicating the interface on two separate regions called contexts, the user can comfortably explore the whole content of a context without the effects of the Midas touch problem. Focus within a context is set by a short dwell time and fast selection is done by switching contexts. We present experimental results for a text entry application with 7 participants that show significant speed improvement over traditional fixed dwell time gaze controlled keyboards. After 8 sessions, 6 participants were able to type about 12 words per minute (wpm), and the fastest participant was able to type above 20 wpm with error rate under 2%.

@inproceedings{Morimoto:ETRA2010,
 author = {Carlos H. Morimoto and Arnon Amir},
 title = {Context Switching for Fast Key Selection in Text Entry Applications},
 booktitle = {Proceedings of the 2010 Symposium on Eye-Tracking Research & Applications},
 series = {ETRA '10},
 year = {2010},
 isbn = {978-1-60558-994-7},
 location = {Austin, Texas},
 pages = {271--274},
 numpages = {4},
 url = {http://doi.acm.org/10.1145/1743666.1743730},
 doi = {10.1145/1743666.1743730},
 acmid = {1743730},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {context switching, gaze interfaces, gaze typing},
}

This paper presents a camera based eye anthropometric measurement system that automatically computes the pupil size, the inter-pupillary distance, palpebral fissure (PF) and the marginal reflex distance (MRD). These values are in general manually obtained in ophthalmologic exams using a millimetric ruler or gauges, and therefore are subject to errors. Besides improving the accuracy and reliability of the measurements, the system uses near-infrared (NIR) light that allows its use in different lighting conditions. A single NIR non-calibrated camera is used to measure both eyes at the same time. The eye features are extracted using image processing algorithms, and metric values are computed using a known chessboard pattern as reference. Experimental results demonstrate the real-time performance, accuracy and robustness of the method.

@inproceedings{Morimoto:WIM2009,
 title = {Automatic Measurement of Eye Features Using Image Processing},
 author = {Carlos H. Morimoto},
 booktitle = {Workshop de Informática Médica},
 year = {2009},
 series = {WIM '09}
}

This paper proposes a method to locate people on a reference plane using multiple cameras. Previous works rely on people trajectories and color models to solve occlusion.This new approach solves people detection under occlusion by accumulating evidence from multiple views instantaneously and does not rely on previous segmentation of individuals in foreground data or any tracking information.First, foreground data from one view, segmented using background subtraction, is projected onto the ground plane or reference image. The projected foreground of a second view overlaps the first projected foreground only on the points where the foreground intersects the ground plane.Thus, by accumulating the evidence from multiple views,people can be located by detecting local maxima on the accumulated reference image. Experimental results using publicly available data from PETSpsila06 [9] show that the method robustly locates people in very challenging situations with occlusion in most of the views. The locations on the ground plane can further be used for segmentation and tracking on each camera view under severe occlusion.

@inproceedings{Santos:SIBGRAPI2010,
 author = {Thiago T. Santos and Carlos H. Morimoto},
 title = {People detection under occlusion in multiple camera views},
 booktitle = {XXI Brazilian Symposium on Computer Graphics and Image Processing},
 series = {SIBGRAPI '08},
 year = {2008},
 publisher = {Sociedade Brasileira de Computação},
 address = {Campo Grande, Brazil},
 pages = {53-60}
}

In this chapter we look at a passive eye monitoring system, or simply eye tracker (ET), as a mono-camera system that measures the position and orientation of the subject’s eye. The most common image based ET technique is known as the pupil-corneal reflection (PCR) technique, because it uses a reflection on the surface of the cornea (the first Purkinje image) generated by an external light source that is used as a “reference” point. The two dimensional image vector defined by the center of the pupil (or the iris) and the corneal reflection is used to estimate the gaze direction, after a simple calibration procedure that defines the mapping from image coordinates to screen coordinates1. To facilitate pupil detection and tracking in real-time, an active differential lighting scheme is commonly used [421]. By placing a light source near the camera optical center, a bright pupil image is generated, while a second light source, distant from the camera center, generates a dark pupil image. The two light sources can be synchronized with the video frame rate, so that at every other frame the camera captures a bright or a dark pupil image, and the pupil can be easily segmented as a high contrast region from the subtraction of the bright and dark pupil images.

@inbook{Morimoto:PEM2008,
 author = {Carlos H. Morimoto and Flávio L. Coutinho},
 editor = {Hammoud, I. Riad},
 title = {Free Head Motion Eye Gaze Tracking Techniques},
 bookTitle = {Passive Eye Monitoring: Algorithms, Applications and Experiments},
 year = {2008},
 publisher = {Springer Berlin Heidelberg},
 address = {Berlin, Heidelberg},
 pages = {133--152},
 isbn = {978-3-540-75412-1},
 doi = {10.1007/978-3-540-75412-1_6},
 url = {http://dx.doi.org/10.1007/978-3-540-75412-1_6}
}

In previous experiments investigating motor control in a Time-To-Contact task [Morya et al., 2003], events occurring 400-600 ms prior to contact (but not earlier or later) caused volunteers to anticipate their estimate of when contact occurred. Many such mislocalization or mistiming effects have been discussed in the literature [Nijhuan, 1994; van Beers et al. 2001]. In preliminary eye-tracking experiments [Morya et al. 2004], with a simplified version of the task, involuntary shifts in gaze suggested the presence of attentional shifts as volunteers prepared to respond, that might be associated with their anticipations. To better understand the factors involved in these observations, gaze was sistematically recorded changing the speed of the moving target, and with different instructions as to where the volunteers should look as they performed the Time-To-Contact task.

@inproceedings{Morya:ETRA2006,
 author = {Edgard Morya and Marco Bertolassi and Adhemar Pettri Filho and Carlos H. Morimoto and Ronald D. Ranvaud},
 title = {Eye Movements and Motor Programming in a Time-To-Contact Task},
 booktitle = {Proceedings of the 2006 Symposium on Eye Tracking Research & Applications},
 series = {ETRA '06},
 year = {2006},
 isbn = {1-59593-305-0},
 location = {San Diego, California},
 pages = {54--54},
 numpages = {1},
 url = {http://doi.acm.org/10.1145/1117309.1117339},
 doi = {10.1145/1117309.1117339},
 acmid = {1117339},
 publisher = {ACM},
 address = {New York, NY, USA},
}

The focus of visual attention is closely related to eye movements and fixations, while episodic memory has been defined as the ability to be consciously aware of an earlier experience [Bond, 2005]. The role of emotional content plays a crucial role in the ability of recalling a previous event. Here we investigate eye fixation patterns and their spatial distribution using ordinary kriging, a geostatistical interpolation method [Canto-Pereira et al., 2005], in a task where emotion and episodic memory were assessed.

@inproceedings{Canto-Pereira:ETRA2006,
 author = {Luiz H. M. do Canto Pereira and Breno Santos and Edgard Morya and Carlos H. Morimoto and Ronald D. Ranvaud},
 title = {The Relation of Eye Fixation Patterns with Emotional Content and Episodic Memory},
 booktitle = {Proceedings of the 2006 Symposium on Eye Tracking Research & Applications},
 series = {ETRA '06},
 year = {2006},
 isbn = {1-59593-305-0},
 location = {San Diego, California},
 pages = {50--50},
 numpages = {1},
 url = {http://doi.acm.org/10.1145/1117309.1117335},
 doi = {10.1145/1117309.1117335},
 acmid = {1117335},
 publisher = {ACM},
 address = {New York, NY, USA},
}

This paper introduces the Gaze based Interface Extensions (GInX) architecture designed for the development of eye-gaze enhanced attentive interfaces. The architecture is composed of 3 modules, the domain, user, and attentive modules. In the absence of information about the user and the domain, the attentive module controls the cursor using gaze and target position information alone. The cursor control can be refined in an attentive way (Vertegaal 2002) as more information about the application and the user are added. The system currently offers 3 different operation modes: Latency, MAGIC, and GInX default mode. In the Latency mode, the cursor position is controlled by gaze and selection is done using dwell time. MAGIC Pointing (Zhai et al. 1999) was suggested to combine the speed of eye tracking with the accuracy of manual pointing devices. GInX extends the concept of Magic Pointing by introducing information about the user and application context in order to eliminate the time required for cursor reacquisition and position adjustment inherent in the original MAGIC Pointing interface. A prototype of GInX was implemented and used to compared the performance of all these 3 modes with a mouse. Our experiments show that GInX outperforms MAGIC Pointing, although the mouse has the best performance overall.

@inproceedings{Barcelos:ETRA2008,
 author = {Thiago S. Barcelos and Carlos H. Morimoto},
 title = {GInX: Gaze Based Interface Extensions},
 booktitle = {Proceedings of the 2008 Symposium on Eye Tracking Research & Applications},
 series = {ETRA '08},
 year = {2008},
 isbn = {978-1-59593-982-1},
 location = {Savannah, Georgia},
 pages = {149--152},
 numpages = {4},
 url = {http://doi.acm.org/10.1145/1344471.1344510},
 doi = {10.1145/1344471.1344510},
 acmid = {1344510},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {gaze aware interfaces},
}

The aim of this paper is to present an automatic eyelid measurement system based on digital video processing techniques. Currently, the protocol to measure the palpebral fissure (PF) and the marginal reflex distance (MRD) requires the use of a millimetric ruler. This procedure is subject to error and the accuracy and reproducibility of the results depend on the experience of the examiner. The computer vision system introduced in this paper uses two near infrared light sources synchronized with the camera to robustly detect and track the pupil, and then segment the limbus and the eyelids. The corneal reflection generated by the light sources are used to create a reference point that is used to define the vertical line along which the measurements are taken, and to determine when the patient is actually looking at the camera. Our experimental results show that the system is robust to the presence of eyelashes, glasses, and contact lenses, and the measurements can be accurate to tenths of millimeters.

@inproceedings{Morimoto:SAC2008,
 author = {Carlos H. Morimoto and Thomaz F. de Oliveira},
 title = {Eyelid Measurements Using Digital Video Processing},
 booktitle = {Proceedings of the 2008 ACM Symposium on Applied Computing},
 series = {SAC '08},
 year = {2008},
 isbn = {978-1-59593-753-7},
 location = {Fortaleza, Ceara, Brazil},
 pages = {1369--1373},
 numpages = {5},
 url = {http://doi.acm.org/10.1145/1363686.1364003},
 doi = {10.1145/1363686.1364003},
 acmid = {1364003},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {eye detection, eye tracking, eyelid measurements},
}

This work introduces Jmin-images as an alternative to gradient maps in watershed-based image segmentation algorithms. A Jmin-image is computed using J-images that are color-texture homogeneity maps based on Fisher’s discriminant, introduced in [1]. The major advantage of using Jmin is the elimination of the scale selection problem for texture segmentation. A filtered Jmin is used as input for a watershed algorithm whose output is refined by a color histogram based hierarchical clustering step. Experimental results show good performance in the segmentation of natural images.

@article{Santos:ISMM2007,
 title = {Jmin-image based color-texture segmentation using watershed and hierarchical clustering},
 author = {Thiago T. Santos and Carlos H. Morimoto and Rama Chellappa},
 journal = {Proceedings of the 8th International Symposium on Mathematical Morphology},
 address = {Rio de Janeiro, Brazil},
 month = {October},
 pages = {35-36},
 year = {2007},
 volume = {2},
}

This paper discusses several usability issues related to the use of gestures as an input mode in multimodal interfaces. The use of gestures has been suggested before as a natural solution for applications that require hands-free and notouch interaction with computers, such as in virtual reality (VR) environments. We introduce a simple but robust 2D computer vision based gesture recognition system that was successfully used for interaction in VR environments such as CAVEs and Powerwalls. This interface was tested under 3 different scenarios, as a regular pointing device in a GUI interface, as a navigation tool, and as a visualization tool. Our experiments show that the time to completion of simple pointing tasks is considerably slower when compared to a mouse and that its use during even short periods of time causes fatigue. Despite, these drawbacks, the use of gestures as an alternative mode in multimodal interfaces offers several advantages, such as quick access to computing resources that might be embedded in the environment, using a natural and intuitive way, and that scales nicely to group and collaborative applications, where gestures can be used sporadically.

@inproceedings{Cabral:CLIHC2005,
 author = {Marcio C. Cabral and Carlos H. Morimoto and Marcelo K. Zuffo},
 title = {On the Usability of Gesture Interfaces in Virtual Reality Environments},
 booktitle = {Proceedings of the 2005 Latin American Conference on Human-computer Interaction},
 series = {CLIHC '05},
 year = {2005},
 isbn = {1-59593-224-0},
 location = {Cuernavaca, Mexico},
 pages = {100--108},
 numpages = {9},
 url = {http://doi.acm.org/10.1145/1111360.1111370},
 doi = {10.1145/1111360.1111370},
 acmid = {1111370},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {computer vision, usability of gesture interfaces, virtual reality},
}

This paper introduces a fast, robust and accurate iris segmentation technique based on active lighting. The geometry of the light sources, a single camera and the eye facilitates the detection of the pupil and the automatic selection of the most appropriate image for biometric identification from the video stream, minimizing the effects of noise, distortion and occlusion during the image acquisition process. Two near infrared (NIR) light sources (that are invisible to the human eye) are synchronized with the video signal. One of the light sources is placed near the optical axis of the camera, and generates a bright pupil image. The second light is placed off-axis, generating dark pupil images. These two images can be easily combined to segment the pupil region, and the corneal reflection of the light sources can be used to select best quality images. The pupil position is then used to segment the iris and eyelids using a coarse-to-fine strategy. Experimental results with a real-time prototype show the quality of the iris segmentation.

@inproceedings{Morimoto:SIBGRAPI2005,
 author = {Carlos H. Morimoto and Thiago T. Santos and Adriano S. Muniz}, 
 booktitle = {XVIII Brazilian Symposium on Computer Graphics and Image Processing (SIBGRAPI'05)}, 
 title = {Automatic Iris Segmentation Using Active Near Infra Red Lighting}, 
 year = {2005}, 
 pages = {37-43}, 
 keywords = {Biometrics;Cameras;Geometry;Image segmentation;Iris;Light sources;Noise robustness;Optical distortion;Optical noise;Streaming media}, 
 doi = {10.1109/SIBGRAPI.2005.14}, 
 ISSN = {1530-1834}, 
 month = {October}
}

This paper introduces JEcoSys, an extensible framework developed in Java to generate interactive simulations of plants. JEcoSys uses a simple physical model of plants, based on dynamic constraints, and pre-computations to achieve realism and efficiency during animation. The framework also includes a force field based technique to model the interaction of plants with different agents, that can simulate natural phenomena such as wind and rain. Experimental results show that JEcosys is able to simulate complex scenes containing hundreds of tufts of grass and flowers in real-time.

@inproceedings{Endo:CGI2005, 
 author = {Luis C. Y. Endo and Carlos H. Morimoto and Antonio E. Fabris}, 
 booktitle = {Computer Graphics International 2005}, 
 title = {JEcoSys - a framework for interactive plants simulation}, 
 year = {2005}, 
 pages = {66-73}, 
 keywords = {Java;computational geometry;computer animation;interactive systems;JEcoSys;Java;computer animation;computer graphics;dynamic constraints;force field based technique;geometric transformations;interactive plants simulation;interactive systems;physically based modeling;Animation;Biological system modeling;Computational modeling;Computer graphics;Interactive systems;Layout;Rain;Real time systems;Rendering (computer graphics);Solid modeling}, 
 doi = {10.1109/CGI.2005.1500376}, 
 ISSN = {1530-1052}, 
 month = {June}
}

This paper presents a review of eye gaze tracking technology and focuses on recent advancements that might facilitate its use in general computer applications. Early eye gaze tracking devices were appropriate for scientific exploration in controlled environments. Although it has been thought for long that they have the potential to become important computer input devices as well, the technology still lacks important usability requirements that hinders its applicability. We present a detailed description of the pupil-corneal reflection technique due to its claimed usability advantages, and show that this method is still not quite appropriate for general interactive applications. Finally, we present several recent techniques for remote eye gaze tracking with improved usability. These new solutions simplify or eliminate the calibration procedure and allow free head motion.

@article{Morimoto:CVIU2005,
 author = {Carlos H. Morimoto and Marcio R. M. Mimica},
 title = {Eye Gaze Tracking Techniques for Interactive Applications},
 journal = {Computer Vision and Image Understanding},
 issue_date = {April 2005},
 volume = {98},
 number = {1},
 month = {apr},
 year = {2005},
 issn = {1077-3142},
 pages = {4--24},
 numpages = {21},
 url = {http://dx.doi.org/10.1016/j.cviu.2004.07.010},
 doi = {10.1016/j.cviu.2004.07.010},
 acmid = {1649095},
 publisher = {Elsevier Science Inc.},
 address = {New York, NY, USA},
}

Panda is the project name for the online Course Management System under development at the Department of Computer Science of IME/USP. The objective of this project is to create an extensible open source framework where e-learning tools can be easily developed, tested and integrated. These tools range from the organization of the courses offered by different institutes of an university, to grading individual assignments submitted by each student. The current platform was developed in Java and is offering support to 29 courses from 6 institutes of the Universisty of São Paulo, with about 3000 students enrolled.

@inproceedings{Morimoto:WEI2004,
 title = {Panda: Sistema de Gerenciamento de Cursos Online},
 author = {Carlos H. Morimoto and Carlos E. Ferreira and José Soares},
 booktitle = {XII Workshop de Educação em Computação},
 series = {WEI '04},
 year = {2004}
}

A system and method for eye gaze tracking without calibrated cameras, direct measurements of specific users' eye geometries, or requiring the user to visually track a cursor traversing a known trajectory. The preferred embodiment includes two uncalibrated cameras imaging the user's eye and haying on-axis lighting. The cameras capture images of a test pattern in real space as reflected from the user's cornea, which is essentially a convex spherical mirror. The invention then extracts parameters required to define a mathematical mapping between real space and image space, including spherical and perspective transformations. The invention processes subsequent images of objects reflected from the user's eye through the inverse of the mathematical mapping to determine a gaze vector and a point of regard. Alternately, a single calibrated camera may be employed with means for estimating the eye-to-camera distance. A head-mounted embodiment that may include a laser pointer is also described.

@misc{Amir:Patent2003,
 title = {System and method for eye gaze tracking using corneal image mapping},
 author = {Arnon Amir and Myron D. Flickner and David B. Koons and Carlos H. Morimoto},
 url = {http://www.google.com/patents/US6659611},
 year = {2003},
 month = {December},
 note = {US Patent 6,659,611}
}

Modeling natural phenomena is a problem that has been studied for a long time in Computer Graphics. A problem that currently arises is how to render and animate realistically natural phenomena in real-time. This paper presents a technique to model underbrush with fast and realistic animation. Pre-computations and a simplified physically based model were developed to achieve the work's proposal.

@inproceedings{Mimica:SIBGRAPI2003, 
 author = {Marcio R. M. Mimica and Carlos H. Morimoto}, 
 booktitle = {XVI Brazilian Symposium on Computer Graphics and Image Processing}, 
 series = {SIBGRAPI '03},
 title = {A computer vision framework for eye gaze tracking}, 
 year = {2003}, 
 pages = {406-412}, 
 keywords = {computer vision;human computer interaction;ray tracing;EGT;calibration function;computer interfaces;computer vision framework;eye gaze tracking;eye model;ray tracing;synthetic images;Computer graphics;Computer interfaces;Computer vision;Costs;Head;Human computer interaction;Image generation;Mice;Ray tracing;Testing}, 
 doi = {10.1109/SIBGRA.2003.1241037}, 
 ISSN = {1530-1834}, 
 month = {October}
}

Eye gaze tracking (EGT) allows us to estimate the direction of gaze and the point of regard. This technique has been successfully used as a pointing device in computer interfaces for people with disabilities, but significant technological advances are still required to make EGT appropriate to be used in general computer interfaces. In order to improve the EGT technology, a testbed where new devices and algorithms can be evaluated must be defined. We present a survey of the methods used for EGT, and organize them into a computer vision framework, that we use to support the development and evaluation of new techniques. As an example of application of the framework, the calibration function used in the EGT developed in our laboratory is tested, and the results show the precision of the method. Higher accuracy of the testing results is achieved using synthetic images generated by ray tracing, from a physically based model of the eye.

@inproceedings{Mimica:SIBGRAPI2003, 
 author = {Marcio R. M. Mimica and Carlos H. Morimoto}, 
 booktitle = {XVI Brazilian Symposium on Computer Graphics and Image Processing}, 
 series = {SIBGRAPI '03},
 title = {A computer vision framework for eye gaze tracking}, 
 year = {2003}, 
 pages = {406-412}, 
 keywords = {computer vision;human computer interaction;ray tracing;EGT;calibration function;computer interfaces;computer vision framework;eye gaze tracking;eye model;ray tracing;synthetic images;Computer graphics;Computer interfaces;Computer vision;Costs;Head;Human computer interaction;Image generation;Mice;Ray tracing;Testing}, 
 doi = {10.1109/SIBGRA.2003.1241037}, 
 ISSN = {1530-1834}, 
 month = {October}
}

Eye gaze tracking (EGT) allows us to estimate the direction of gaze and the point of regard. This technique has been successfully used as a pointing device in computer interfaces for people with disabilities, but significant technological advances are still required to make EGT appropriate to be used in general computer interfaces. In order to improve the EGT technology, a testbed where new devices and algorithms can be evaluated must be defined. We present a survey of the methods used for EGT, and organize them into a computer vision framework, that we use to support the development and evaluation of new techniques. As an example of application of the framework, the calibration function used in the EGT developed in our laboratory is tested, and the results show the precision of the method. Higher accuracy of the testing results is achieved using synthetic images generated by ray tracing, from a physically based model of the eye.

@misc{Flickner:Patent2003,
 title = {Method and system for relevance feedback through gaze tracking and ticker interfaces},
 author = {Myron D. Flickner and David B. Koons and Qi Lu and Paul P. Maglio and Carlos H. Morimoto and Edwin J. Selker},
 url = {http://www.google.com/patents/US6577329},
 year = {2003},
 month = {June},
 note = {US Patent 6,577,329}
}

A system and method (and signal medium) for interactively displaying information, include a ticker display for displaying items having different views, a tracker for tracking a user's eye movements while observing a first view of information on the ticker display, and a mechanism, based on an output form the tracker, for determining whether a current view has relevance to the user.

@article{Endo:WSCG2003,
 title = {Real-time animation of underbrush},
 author = {Luis C. Y. Endo and Carlos H. Morimoto and Antonio E. Fabris},
 journal = {WSCG SHORT PAPERS proceedings},
 volume = {11},
 number = {1},
 year = {2003},
 month = {February},
 publisher = {UNION Agency - Science Press},
 issn = {1213-6972}
}

We introduce a new method for computing the 3D position of an eye and its gaze direction from a single camera and at least two near infra-red light sources. The method is based on the theory of spherical optical surfaces and uses the Gullstrand model of the eye to estimate the positions of the center of the cornea and the center of the pupil in 3D. The direction of gaze can then be computed from the vector connecting these two points. The point of regard can also be computed from the intersection of the direction of gaze with an object in the scene. We have simulated this model using ray traced images of the eye, and obtained very promising results. The major contribution of this new technique over current eye tracking technology is that the system does not require to be calibrated with the user before each user session, and it allows for free head motion.

@inproceedings{Morimoto:ICPR2002, 
 author = {Carlos H. Morimoto and Arnon Amir and Myron Flickner}, 
 booktitle = {Proceedings of the 16th International Conference on Pattern Recognition}, 
 title = {Detecting eye position and gaze from a single camera and 2 light sources}, 
 year = {2002}, 
 volume = {4}, 
 pages = {314-317 vol.4}, 
 keywords = {eye;optical tracking;physiological models;ray tracing;Gullstrand model;cornea center position;eye position detection;eye tracking technology;free head motion;gaze direction computation;human computer interaction;infra-red light sources;pupil center position;ray traced images;single camera;spherical optical surfaces;Calibration;Cameras;Computer vision;Cornea;Head;Human computer interaction;Layout;Light sources;Optical sensors;Tracking}, 
 doi = {10.1109/ICPR.2002.1047459}, 
 ISSN = {1051-4651}, 
 month = {August}
}

A method and apparatus determine when a subject is looking at a specific target area by estimating a divergence angle between (1) the direction in which the subject is looking and (2) the direction from the subject directly to the target area. This technique accesses whether the subject is looking at a particular area. The invention may further condition this determination according to the subject's distance from the target area, because there is less tolerance for divergent angles when the subject is farther away. In one embodiment, the divergence angle is estimated using the position of a glint of light in the subject's pupil. The glint is created by a light source located in the target area. If the glint is sufficiently central to the pupil, with the camera and light source being near the target area, the subject is looking at the target area. At long distances, when the glint is not sufficiently discernable from the pupil, another technique may be employed to estimate divergence angle. Namely, the plane of the subject's face is computed, and analyzed with respect to a vector between the subject's face and the target area. If the plane is substantially normal to the vector, the subject is looking at the target area.

@misc{Amir:Patent2002,
 title = {Method and apparatus for determining eye contact},
 author = {Arnon Amir and Myron D. Flickner and David B. Koons and Carlos H. Morimoto and Shumin Zhai},
 url = {http://www.google.st/patents/US6393136},
 year = {2002},
 month = {May},
 note = {US Patent 6,393,136}
}

This paper introduces a novel technique for remote eye gaze tracking and detection of point of regard that is specially designed for wide use in HCI. It addresses and eliminates two of the major problems of commercial remote eye gaze tracking, namely the need for user calibration before each session and of accuracy degradation with head movement. The new technique uses a single calibrated camera, several light sources with known positions and a physical model of the eye to estimate the 3D position of the eye and its gaze direction. Simulation results using ray tracing are used to study the accuracy and robustness of the system, and demonstrate its operability.

@inproceedings{Morimoto:CHI2002,
 author = {Carlos H. Morimoto and Arnon Amir and Myron D. Flickner},
 title = {Free Head Motion Eye Gaze Tracking Without Calibration},
 booktitle = {CHI '02 Extended Abstracts on Human Factors in Computing Systems},
 series = {CHI EA '02},
 year = {2002},
 isbn = {1-58113-454-1},
 location = {Minneapolis, Minnesota, USA},
 pages = {586--587},
 numpages = {2},
 url = {http://doi.acm.org/10.1145/506443.506496},
 doi = {10.1145/506443.506496},
 acmid = {506496},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {gaze based interfaces, remote eye gaze tracking},
}

This paper describes some preliminary results of our ongoing project on digital mirror interfaces. Regular mirrors reflect ambient light from the scene towards the observer. Digital mirrors capture the ambient light with a camera, extract information about the scene, and display appropriate information to the user, combining real-time computer vision systems with realistic computer graphics. We describe the works on both ends: image processing from the camera, and image output to a computer screen. The computer vision routines developed so far are basically for human face detection, tracking and 3D head pose estimation. The main computer graphics routines are able to render a 3D head model in real-time. We integrate the both ends, coding very simple behaviors to the virtual head.

@inproceedings{Morimoto:SIBGRAPI2001,
 author = {Carlos H. Morimoto},
 title = {Interactive Digital Mirror},
 booktitle = {Proceedings of the 14th Brazilian Symposium on Computer Graphics and Image Processing},
 series = {SIBGRAPI '01},
 year = {2001},
 isbn = {0-7695-1330-1},
 pages = {232--236},
 numpages = {5},
 url = {http://dl.acm.org/citation.cfm?id=646015.677943},
 acmid = {677943},
 publisher = {IEEE Computer Society},
 address = {Washington, DC, USA},
}

A system for secure data entry, includes a virtual keypad having a plurality of keys, a mechanism for determining to which virtual key of the keypad a user is looking, and an actuator, operable by the user, for confirming key selection. Another system for secure data entry, includes a virtual keypad having a plurality of keys, an eye tracker for tracking eye movement of a user, and for receiving a coded input from an eye movement of the user gazing upon at least a selected one of the keys of the virtual keypad, and an actuator for being selectively actuated by the user upon confirmation of the coded input by the user.

@misc{Flickner:Patent2001,
 title = {Gaze-based secure keypad entry system},
 author = {Myron D. Flickner and Qi Lu and Carlos H. Morimoto},
 url = {http://www.google.ch/patents/US6282553},
 year = {2001},
 month = {August},
 note = {US Patent 6,282,553}
}

This paper presents our ongoing work in developing virtual autonomous agents to facilitate human computer interaction, and in particular, perceptual user interfaces (PUIs) for desktop applications based on computer vision techniques. The desktop environment sufficiently constrains the computer vision problem in order to allow robust real-time performance for our agents. Currently the computer vision module is able to detect and track faces in real-time, and broadcast their positions to autonomous software agents. We are using a synthetic face to visualize the behaviors of the autonomous agent, which are very simple so far, and to give the user graphic feedback of the agent’s status. Basically the agent detects the presence of the user by showing a happy face expression, tracks the user’s face, becomes sad once the user is gone, and angry if s/he is gone for a long time. Extensions to the vision module to track facial features and recognize facial expressions, as well as extensions to the agent’s behaviors, are being implemented.

@inproceedings{Morimoto:WAICV00,
 title = {Virtual autonomous agents with vision},
 author = {Carlos H. Morimoto},
 booktitle = {Proceedings of the Workshop on Artificial Intelligence and Computer Vision},
 series = {I WAICV},
 year = {2000}
}

This paper presents a multiple face detector based on a robust pupil detection technique. The pupil detector uses active illumination that exploits the retro-reflectivity property of eyes to facilitate detection. The detection range of this method is appropriate for interactive desktop and kiosk applications. Once the location of the pupil candidates are computed, the candidates are filtered and grouped into pairs that correspond to faces using heuristic rules. To demonstrate the robustness of the face detection technique, a dual-mode face tracker was developed, which is initialized with the most salient detected face. Recursive estimators are used to guarantee the stability of the process and combine the measurements from the multi-face detector and a feature correlation tracker. The estimated position of the face is used to control a pan-tilt servo mechanism in real-time, that moves the camera to keep the tracked face always centered in the image

@inproceedings{Morimoto:ICAFGR2000, 
 author = {Carlos H. Morimoto and Myron D. Flickner}, 
 booktitle = {Proceedings of the Fourth IEEE International Conference on Automatic Face and Gesture Recognition},
 title = {Real-time multiple face detection using active illumination}, 
 year = {2000}, 
 pages = {8-13}, 
 keywords = {eye;face recognition;feature extraction;filtering theory;real-time systems;recursive estimation;recursive filters;tracking;active illumination;dual-mode face tracker;eyes;feature correlation tracker;filtering;heuristic rules;interactive desktop applications;kiosk applications;multiple face detection;pan-tilt servo mechanism;position estimation;pupil detection;real-time control;recursive estimators;retro-reflectivity;Cameras;Computer vision;Detectors;Eyes;Face detection;Lighting;Recursive estimation;Robustness;Servomechanisms;Stability}, 
 doi = {10.1109/AFGR.2000.840605}, 
 month = {March}
}

We present a fast, robust, and low cost pupil detection technique that uses two near-infrared time multiplexed light sources synchronized with the camera frame rate. The two light sources generate bright and dark pupil images, which are used for pupil segmentation. To reduce artifacts caused mostly by head motion, a larger temporal support is used. This method can be applied to detect and track several pupils (or several people). Experimental results from a real-time implementation of the system show that this technique is very robust, and able to detect pupils using wide field of view low cost cameras under different illumination conditions, even for people with glasses, from considerable long distances.

@article{Morimoto:IVC2000,
 title = {Pupil detection and tracking using multiple light sources},
 journal = {Image and Vision Computing},
 volume = {18},
 number = {4},
 pages = {331-335},
 year = {2000},
 issn = {0262-8856},
 doi = {http://dx.doi.org/10.1016/S0262-8856(99)00053-0},
 url = {http://www.sciencedirect.com/science/article/pii/S0262885699000530},
 author = {Carlos H. Morimoto and David B. Koons and Arnon Amir and Myron D. Flickner},
 keywords = {Pupil detection, Gaze tracking}
}

Advanced human-computer interaction (HCI) techniques are required to enhance current computer interfaces. In this paper, we present an eye gaze tracking system based on a robust low-cost real-time pupil detector, and we describe some eye-aware applications that are being developed to enhance HCI. Images of pupils are segmented using an active lighting scheme that exploits very particular properties of eyes. Once the pupil is detected, its center is tracked, along with the corneal reflection (CR) generated by the light sources. Assuming little head motion, the eye gaze direction is computed based on the vector between the centers of the CR and the pupil, after a brief calibration procedure. Other information, such as pupil size and blink rate, can also be made available. The current prototype runs at frame rate, providing 30 samples of the gaze position per second to gaze-aware applications, such as advanced pointing and selection mechanisms.

@inproceedings{Morimoto:SIBGRAPI1999, 
 author = {Carlos H. Morimoto and David B. Koons and Arnon Amir and Myron D. Flickner and Shumin Zhai}, 
 booktitle = {Proceedings of the XII Brazilian Symposium on Computer Graphics and Image Processing}, 
 title = {Keeping an eye for HCI}, 
 year = {1999}, 
 pages = {171-176}, 
 keywords = {calibration;eye;feature extraction;image processing equipment;image segmentation;interactive devices;light reflection;lighting;optical tracking;pointing systems;real-time systems;user interfaces;active lighting scheme;blink rate;calibration procedure;computer interfaces;corneal reflection;eye gaze direction;eye gaze tracking system;eye-aware applications;frame rate;gaze-aware applications;head motion;human-computer interaction;light sources;pointing mechanisms;pupil centre tracking;pupil image segmentation;pupil size;real-time pupil detector;selection mechanisms;Application software;Chromium;Computer interfaces;Detectors;Eyes;Human computer interaction;Image segmentation;Optical reflection;Real time systems;Robustness}, 
 doi = {10.1109/SIBGRA.1999.805722}, 
 month = {October}
}

We present a robust, frame-rate pupil detector technique, based on an active illumination scheme, used for gaze estimation. The pupil detector uses two light sources synchronized with the even and odd fields of the video signal (interlaced frames), to create bright and dark pupil images. The retro-reflectivity property of the eye is exploited by placing an infra-red (IR) light source close to the camera's optical axis resulting in an image with a bright pupil. A similar off axis IR source generates an image with dark pupils. Pupils are detected from the thresholded difference of the bright and dark pupil images. After a calibration procedure, the vector computed from the pupil center to the center of the corneal glints generated from light sources is used to estimate the gaze position. The frame-rate gaze estimator prototype is currently being demonstrated in a docked 300 MHz IBM Thinkpad with a PCI frame grabber, using interlaced frames of resolution 640 480 8 bits.

@inproceedings{Morimoto:ICCV1999,
 author = {Carlos H. Morimoto and David B. Koons and Arnon Amir and Myron D. Flickner},
 title = {Frame-rate pupil detector and gaze tracker},
 booktitle = {Proceedings of the IEEE ICCV'99 frame-rate workshop},
 year = {1999},
 month = {September}
}

This work explores a new direction in utilizing eye gaze for computer input. Gaze tracking has long been considered as an alternative or potentially superior pointing method for computer input. We believe that many fundamental limitations exist with traditional gaze pointing. In particular, it is unnatural to overload a perceptual channel such as vision with a motor control task. We therefore propose an alternative approach, dubbed MAGIC (Manual And Gaze Input Cascaded) pointing. With such an approach, pointing appears to the user to be a manual task, used for fine manipulation and selection. However, a large portion of the cursor movement is eliminated by warping the cursor to the eye gaze area, which encompasses the target. Two specific MAGIC pointing techniques, one conservative and one liberal, were designed, analyzed, and implemented with an eye tracker we developed. They were then tested in a pilot study. This early- stage exploration showed that the MAGIC pointing techniques might offer many advantages, including reduced physical effort and fatigue as compared to traditional manual pointing, greater accuracy and naturalness than traditional gaze pointing, and possibly faster speed than manual pointing. The pros and cons of the two techniques are discussed in light of both performance data and subjective reports.

@inproceedings{Zhai:CHI1999,
 author = {Shumin Zhai and Carlos H. Morimoto and Steven Ihde},
 title = {Manual and Gaze Input Cascaded (MAGIC) Pointing},
 booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
 series = {CHI '99},
 year = {1999},
 isbn = {0-201-48559-1},
 location = {Pittsburgh, Pennsylvania, USA},
 pages = {246--253},
 numpages = {8},
 url = {http://doi.acm.org/10.1145/302979.303053},
 doi = {10.1145/302979.303053},
 acmid = {303053},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {Fitts'law, computer input, computer vision, eye, eye tracking, gaze, gaze tracking, multi-modal interface, pointing},
}

Perceptual user interfaces will require the detection, tracking, and recognition of faces and other body and facial features. This paper introduces a robust, accurate, and low cost real-time solution for the eye and face detection problem. The method uses two infra-red illumination sources to generate bright and dark pupil images, which are combined to robustly detect pupils. Once the pupils are detected, the inter-ocular distance is used to determine the size and position of the bounding box around the face. The position of other facial features such as eye brows, nose, and mouth can be estimated once the face is detected. A real-time implementation of the system, which process 30 frames per second using interlaced images of resolution 640x480 pixels, is also presented.

@inproceedings{Morimoto:PUI1998,
 author = {Carlos H. Morimoto and David B. Koons and Arnon Amir and Myron D. Flickner},
 title = {Real-time detection of eyes and faces},
 booktitle = {Workshop on Perceptual User Interfaces},
 pages = {117-120},
 year = {1998}
}

Several techniques for electronic image stabilization have been proposed, but very little research has been done to compare and evaluate such techniques. We propose a set of measures to evaluate image stabilization algorithms based of their fidelity, displacement range, and performance. These measures do not require calibration or ground truth, making the evaluation procedure very simple and flexible, i.e., it provides the means to compare techniques based on different motion models. We have used this procedure to compare several image stabilization algorithms and also evaluate the sensitivity of these algorithms to some of its parameters. These same procedures could also be used for the comparison and evaluation of motion estimation and image registration techniques.

@inproceedings{Morimoto:ASSP1998, 
 author = {Carlos H. Morimoto and Rama Chellappa}, 
 booktitle = {Proceedings of the 1998 IEEE International Conference on Acoustics, Speech and Signal Processing},
 title = {Evaluation of image stabilization algorithms}, 
 year = {1998}, 
 volume = {5}, 
 pages = {2789-2792 vol.5}, 
 keywords = {image registration;image sequences;motion compensation;motion estimation;stability;video signal processing;compensated video sequence;displacement range;electronic image stabilization;fidelity;image registration;image stabilization algorithms;motion estimation;motion models;performance;Calibration;Cameras;Displacement measurement;Military computing;Motion estimation;Motion measurement;PSNR;Pixel;Signal to noise ratio;Video sequences}, 
 doi = {10.1109/ICASSP.1998.678102}, 
 ISSN = {1520-6149}, 
 month = {May}
}

We propose a novel video coding scheme to improve the performance of established block-based motion compensation codecs such as MPEG, H261, and H263. The proposed method is a hybrid scheme which introduces model-based global motion compensation as a pre-processing step to the basic block-based motion compensation technique. Performance evaluation tests show that the new method is capable of achieving higher compression rates with the addition of a very small overhead due to global motion estimation. In this paper we describe a codec based on the hybrid motion compensation technique in the context of H263, and present experimental results, comparing the performance of different codecs using rate-distortion curves for several test sequences.

@inproceedings{Morimoto:ICIP1997, 
 author = {Carlos H. Morimoto and Philippe Burlina and Rama Chellappa}, 
 booktitle = {Proceedings of the International Conference on Image Processing}, 
 title = {Video coding using hybrid motion compensation}, 
 year = {1997}, 
 volume = {1}, 
 pages = {89-92 vol.1}, 
 keywords = {data compression;image sequences;motion compensation;motion estimation;rate distortion theory;telecommunication standards;video codecs;video coding;H261;H263;MPEG;block-based motion compensation codecs;block-based motion compensation technique;codec;compression rates;global motion estimation;hybrid motion compensation;model-based global motion compensation;pre-processing step;rate-distortion curves;test sequences;video coding scheme;Automation;Code standards;Codecs;Image coding;Image segmentation;Layout;Motion compensation;Motion estimation;Testing;Video coding}, 
 doi = {10.1109/ICIP.1997.647391}, 
 month = {October}
}

The authors present a fast electronic image stabilization system that compensates for 3D rotation. The extended Kalman filter framework is employed to estimate the rotation between frames, which is represented using unit quaternions. A small set of automatically selected and tracked feature points are used as measurements. The effectiveness of this technique is also demonstrated by constructing mosaic images from the motion estimates, and comparing them to mosaics built from 2D stabilization algorithms. Two different stabilization schemes are presented. The first, implemented in a real-time platform based on a Datacube MV200 board, estimates the motion between two consecutive frames and is able to process gray level images of resolution 128×120 at 10 Hz. The second scheme estimates the motion between the current frame and an inverse mosaic; this allows better estimation without the need for indexing the new image frames. Experimental results for both schemes using real and synthetic image sequences are presented.

@inproceedings{Morimoto:CVPR1997, 
 author = {Carlos H. Morimoto and Rama Chellappa}, 
 booktitle = {Proceedings of the 1997 IEEE Computer Society Conference on Computer Vision and Pattern Recognition}, 
 title = {Fast 3D stabilization and mosaic construction}, 
 year = {1997}, 
 pages = {660-665}, 
 keywords = {Kalman filters;computer vision;feature extraction;image sequences;motion estimation;real-time systems;rotation;stability;10 Hz;2D stabilization algorithms;3D rotation compensation;Datacube MV200 board;automatically selected feature points;automatically tracked feature points;extended Kalman filter framework;fast 3D stabilization;fast electronic image stabilization system;gray level image processing;image frame indexing;inverse mosaic;mosaic image construction;real image sequences;real-time platform;rotation estimation;synthetic image sequences;unit quaternions;Cameras;Image motion analysis;Image resolution;Image sequence analysis;Image sequences;Indexing;Motion analysis;Motion estimation;Pixel;Robot vision systems}, 
 doi = {10.1109/CVPR.1997.609396}, 
 ISSN = {1063-6919}, 
 month = {June}
}

We present two fast implementations of electronic image stabilization and mosaicking systems. The rst one is based on a 2D similarity model and is targeted to process PREDATOR video data. The second system uses a 3D model and compensates for 3D rotation. Both systems have been implemented on parallel pipeline imageprocessing hardware (a Datacube Max-Video 200) connected to a Themis 10MP. Both algorithms use a featurebased multi-resolution technique which tracks a small set of features to estimate the motion of the camera. The extended Kalman filter framework is employed by the 3D de rotation system. The inter-frame motion estimates relative to a reference frame are used to warp the current frame in order to achieve stabilization. The estimates are also used to construct mosaics by aligning the frames. A fast mosaicking implementation is presented for the 2D system. Experimental results demonstrate the robustness of both systems at frame rates above 10 frames/second.

@inproceedings{Morimoto:IUW1997,
 author = {Carlos H. Morimoto and Rama Chellappa and Steve Balakirsky},
 title = {Fast Image Stabilization and Mosaicking},
 year = {1997},
 booktitle = {DARPA Image Understanding Workshop},
 series = {IUW '97}
}

This report describes accomplishments on a UGV RSTA project conducted by a consortium led by the University of Maryland and including the University of Pennsylvania, the University of Rochester, and the National Institute of Standards and Technology. We first review work done on the design, implementation and integration of real time vision algorithms for image stabilization, detection of moving objects from a moving platform and camera control. We then present brief descriptions of a number of supporting basic research projects conducted by the members of the consortium.

@techreport{Davis:Tech1996,
 title = {RSTA on the Move: Detection and Tracking of Moving Objects from an Autonomous Mobile Platform},
 author = {Larry S. Davis and Carlos H. Morimoto and Martin Herman and Ruzena Bajcsy and Randal Nelson},
 year = {1996},
 institution = {Maryland Univ. College Park Center for Automation Research},
}

We study the performance associated with model-based video coding schemes using global motion models for motion compensation. Reference frames and compensated frame dierences are coded using a method similar to MPEG, employing transform coding, quantization, and Human coding. The traditional block matching motion compensation approach is compared to global motion compensation approaches derived from 3D motion stabilization methods, using similarity and pro jective transformations. 3D model-based motion compensation is achieved by derotating the input sequence. This yields a pro jective transformation, which under certain circumstances, is well approximated by a similarity transformation. Experiments are carried out to analyze the performance of each scheme for comparable coding rates.

@techreport{Morimoto:Tech1996,
 author = {Carlos H. Morimoto and Philippe Burlina and Rama Chellappa and Yi-Sheng Yao},
 title = {Video Coding by Model-Based Stabilization},
 institution = {Maryland Univ. College Park Center for Automation Research},
 year = {1996}
}

We present two different approaches for the identification of independently moving objects (IMOs) and demonstrate their application to outdoor imagery taken from a moving autonomous vehicle. Both approaches involve image stabilization followed by an analysis of the stabilized image sequence. The stabilization reduces the effects of the movement of the autonomous vehicle, facilitating the detection of the IMOs. In the first approach, IMOs are detected based on a filtering approach that integrates the results of velocity tuned filters over several frames. In the second approach IMOs are identified by constraints on allowable values of the optic flow field after stabilization.

@inproceedings{Morimoto:IVW1995,
 author = {Carlos H. Morimoto and Daniel Dementhon and Larry Davis and Rama Chellappa and Randal Nelson},
 title = {Detection of Independently Moving Objects in Passive Video},
 booktitle = {In Proceedings of Intelligent Vehicles Workshop},
 year = {1995},
 pages = {270--275}
}

We improve the performance of conventional motion compensated discrete cosine transform video coding. For motion compensation, we employ a two step algorithm in which the camera motion is compensated first and then the motion of moving objects is estimated. We use a feature matching algorithm for camera motion compensation. Motion compensated frame differences are divided into three regions called stationary background, moving objects, and newly emerging area. A region adaptive subband image coding scheme is used for spatial coding of these regions.

@inproceedings{Kwon:ICASSP1995, 
 author = {Oh-Jin Kwon and Rama Chellappa and Carlos H. Morimoto}, 
 booktitle = {International Conference on Acoustics, Speech, and Signal Processing}, 
 series = {ICASSP '95},
 title = {Motion compensated subband coding of video acquired from a moving platform}, 
 year = {1995}, 
 volume = {4}, 
 pages = {2185-2188 vol.4}, 
 keywords = {discrete cosine transforms;feature extraction;motion compensation;motion estimation;transform coding;video cameras;video coding;camera motion compensation;discrete cosine transform;feature matching algorithm;motion compensated frame differences;motion compensated subband coding;motion compensation;motion estimation;moving objects;moving platform;newly emerging area;region adaptive subband image coding;spatial coding;stationary background;two step algorithm;video coding;Automation;Cameras;Decoding;Discrete cosine transforms;Image coding;Layout;Lighting;Motion compensation;Motion estimation;Video coding}, 
 doi = {10.1109/ICASSP.1995.479909}, 
 ISSN = {1520-6149}, 
 month = {May}
}