% % This file was created by the Typo3 extension % sevenpack version 0.7.14 % % --- Timezone: CEST % Creation date: 2013-06-19 % Creation time: 04-07-44 % --- Number of references % 92 % @Article { vonSaldernN2013, title = {Sensory and Striatal Areas Integrate Auditory and Visual Signals into Behavioral Benefits during Motion Discrimination}, journal = {Journal of Neuroscience}, year = {2013}, month = {5}, volume = {33}, number = {20}, pages = {8841-8849}, abstract = {For effective interactions with our dynamic environment, it is critical for the brain to integrate motion information from the visual and auditory senses. Combining fMRI and psychophysics, this study investigated how the human brain integrates auditory and visual motion into benefits in motion discrimination. Subjects discriminated the motion direction of audiovisual stimuli that contained directional motion signal in the auditory, visual, audiovisual, or no modality at two levels of signal reliability. Therefore, this 2 \(\times\) 2 \(\times\) 2 factorial design manipulated: (1) auditory motion information (signal vs noise), (2) visual motion information (signal vs noise), and (3) reliability of motion signal (intact vs degraded). Behaviorally, subjects benefited significantly from audiovisual integration primarily for degraded auditory and visual motion signals while obtaining near ceiling performance for “unisensory” signals when these were reliable and intact. At the neural level, we show audiovisual motion integration bilaterally in the visual motion areas hMT+/V5+ and implicate the posterior superior temporal gyrus/planum temporale in auditory motion processing. Moreover, we show that the putamen integrates audiovisual signals into more accurate motion discrimination responses. Our results suggest audiovisual integration processes at both the sensory and response selection levels. In all of these regions, the operational profile of audiovisual integration followed the principle of inverse effectiveness, in which audiovisual response suppression for intact stimuli turns into response enhancements for degraded stimuli. This response profile parallels behavioral indices of audiovisual integration, in which subjects benefit significantly from audiovisual integration only for the degraded conditions.}, department = {Research Group Noppeney}, web_url = {http://www.jneurosci.org/content/33/20/8841.full.pdf+html}, DOI = {10.1523/​JNEUROSCI.3020-12.2013}, author = {von Saldern, S and Noppeney, U} } @Article { LeitaoTWPN2012, title = {Effects of Parietal TMS on Visual and Auditory Processing at the Primary Cortical Level: A Concurrent TMS-fMRI Study}, journal = {Cerebral Cortex}, year = {2013}, month = {4}, volume = {23}, number = {4}, pages = {873-884}, abstract = {Accumulating evidence suggests that multisensory interactions emerge already at the primary cortical level. Specifically, auditory inputs were shown to suppress activations in visual cortices when presented alone but amplify the blood oxygen level–dependent (BOLD) responses to concurrent visual inputs (and vice versa). This concurrent transcranial magnetic stimulation–functional magnetic resonance imaging (TMS-fMRI) study applied repetitive TMS trains at no, low, and high intensity over right intraparietal sulcus (IPS) and vertex to investigate top-down influences on visual and auditory cortices under 3 sensory contexts: visual, auditory, and no stimulation. IPS-TMS increased activations in auditory cortices irrespective of sensory context as a result of direct and nonspecific auditory TMS side effects. In contrast, IPS-TMS modulated activations in the visual cortex in a state-dependent fashion: it deactivated the visual cortex under no and auditory stimulation but amplified the BOLD response to visual stimulation. However, only the response amplification to visual stimulation was selective for IPS-TMS, while the deactivations observed for IPS- and Vertex-TMS resulted from crossmodal deactivations induced by auditory activity to TMS sounds. TMS to IPS may increase the responses in visual (or auditory) cortices to visual (or auditory) stimulation via a gain control mechanism or crossmodal interactions. Collectively, our results demonstrate that understanding TMS effects on (uni)sensory processing requires a multisensory perspective.}, department = {Research Group Noppeney}, department2 = {Department Scheffler}, web_url = {http://cercor.oxfordjournals.org/content/23/4/873.full.pdf+html}, DOI = {10.1093/cercor/bhs078}, author = {Leit{\~a}o, J and Thielscher, A and Werner, S and Pohmann, R and Noppeney, U} } @Article { RoheWF2012, title = {Dissociation of BOLD responses to reward prediction errors and reward receipt by a model comparison}, journal = {European Journal of Neuroscience}, year = {2012}, month = {8}, volume = {36}, number = {3}, pages = {2376–2382}, abstract = {The representation of reward anticipation and reward prediction errors is the basis for reward-associated learning. The representation of whether or not a reward occurred (reward receipt) is important for decision making. Recent studies suggest that, while reward anticipation and reward prediction errors are encoded in the midbrain and the ventral striatum, reward receipts are encoded in the medial orbitofrontal cortex. In order to substantiate this functional specialization we analyzed data from an fMRI study in which 59 subjects completed two simple monetary reward paradigms. Because reward receipts and reward prediction errors were correlated, a statistical model comparison was applied separating the effects of the two. Reward prediction error fitted BOLD responses significantly better than reward receipt in the midbrain and the ventral striatum. Conversely, reward receipt fitted BOLD responses better in the orbitofrontal cortex. Activation related to reward anticipation was found in the orbitofrontal cortex. The results confirm a functional specialization of behaviorally important aspects of reward processing within the mesolimbic dopaminergic system.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://onlinelibrary.wiley.com/doi/10.1111/j.1460-9568.2012.08125.x/pdf}, DOI = {10.1111/j.1460-9568.2012.08125.x}, author = {Rohe, T and Weber, B and Fliessbach, K} } @Article { ConradVN2012, title = {Interactions Between Apparent Motion Rivalry in Vision and Touch}, journal = {Psychological Science}, year = {2012}, month = {8}, volume = {23}, number = {8}, pages = {940-948}, abstract = {In multistable perception, the brain alternates between several perceptual explanations of ambiguous sensory signals. It is unknown whether multistable processes can interact across the senses. In the study reported here, we presented subjects with unisensory (visual or tactile), spatially congruent visuotactile, and spatially incongruent visuotactile apparent motion quartets. Congruent stimulation induced pronounced visuotactile interactions, as indicated by increased dominance times for both vision and touch, and an increased percentage bias for the percept already dominant under unisensory stimulation. Thus, the joint evidence from vision and touch stabilizes the more likely perceptual interpretation and thereby decelerates the rivalry dynamics. Yet the temporal dynamics depended also on subjects’ attentional focus and was generally slower for tactile than for visual reports. Our results support Bayesian approaches to perceptual inference, in which the probability of a perceptual interpretation is determined by combining visual, tactile, or visuotactile evidence with modality-specific priors that depend on subjects’ attentional focus. Critically, the specificity of visuotactile interactions for spatially congruent stimulation indicates multisensory rather than cognitive-bias mechanisms.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Ernst}, department3 = {Research Group Noppeney}, web_url = {http://pss.sagepub.com/content/23/8/940.full.pdf+html}, DOI = {10.1177/0956797612438735}, author = {Conrad, V and Vitello, MP and Noppeney, U} } @Article { GianiOBKPN2012, title = {Steady-state responses in MEG demonstrate information integration within but not across the auditory and visual senses}, journal = {NeuroImage}, year = {2012}, month = {4}, volume = {60}, number = {2}, pages = {1478–1489}, abstract = {To form a unified percept of our environment, the human brain integrates information within and across the senses. This MEG study investigated interactions within and between sensory modalities using a frequency analysis of steady-state responses that are elicited time-locked to periodically modulated stimuli. Critically, in the frequency domain, interactions between sensory signals are indexed by crossmodulation terms (i.e. the sums and differences of the fundamental frequencies). The 3x2 factorial design, manipulated (1) modality: auditory, visual or audiovisual (2) steady-state modulation: the auditory and visual signals were modulated only in one sensory feature (e.g. visual gratings modulated in luminance at 6 Hz) or in two features (e.g. tones modulated in frequency at 40 Hz \& amplitude at 0.2 Hz). This design enabled us to investigate crossmodulation frequencies that are elicited when two stimulus features are modulated concurrently (i) in one sensory modality or (ii) in auditory and visual modalities. In support of within-modality integration, we reliably identified crossmodulation frequencies when two stimulus features in one sensory modality were modulated at different frequencies. In contrast, no crossmodulation frequencies were identified when information needed to be combined from auditory and visual modalities. The absence of audiovisual crossmodulation frequencies suggests that the previously reported audiovisual interactions in primary sensory areas may mediate low level spatiotemporal coincidence detection that is prominent for stimulus transients but less relevant for sustained SSR responses. In conclusion, our results indicate that information in SSRs is integrated over multiple time scales within but not across sensory modalities at the primary cortical level.}, department = {Research Group Noppeney}, department2 = {Department B{\"u}lthoff}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811912001322}, DOI = {10.1016/j.neuroimage.2012.01.114}, author = {Giani, AS and Ortiz, EB and Belardinelli, P and Kleiner, M and Preissl, H and Noppeney, U} } @Article { HelbigERPTMSN2011, title = {The neural mechanisms of reliability weighted integration of shape information from vision and touch}, journal = {NeuroImage}, year = {2012}, month = {4}, volume = {60}, number = {2}, pages = {1063–1072}, abstract = {Behaviourally, humans have been shown to integrate multisensory information in a statistically-optimal fashion by averaging the individual unisensory estimates according to their relative reliabilities. This form of integration is optimal in that it yields the most reliable (i.e. least variable) multisensory percept. The present study investigates the neural mechanisms underlying integration of visual and tactile shape information at the macroscopic scale of the regional BOLD response. Observers discriminated the shapes of ellipses that were presented bimodally (visual-tactile) or visually alone. A 2\(\times\)5 factorial design manipulated (i) the presence vs. absence of tactile shape information and (ii) the reliability of the visual shape information (five levels). We then investigated whether regional activations underlying tactile shape discrimination depended on the reliability of visual shape. Indeed, in primary somatosensory cortices (bilateral BA2) and the superior parietal lobe the responses to tactile shape input were increased when the reliability of visual shape information was reduced. Conversely, tactile inputs suppressed visual activations in the right posterior fusiform, when the visual signal was blurred and unreliable. Somatosensory and visual cortices may sustain integration of visual and tactile shape information either via direct connections from visual areas or top-down effects from higher order parietal areas.}, department = {Research Group Ernst}, department2 = {Department Scheffler}, department3 = {Department Sch{\"o}lkopf}, department4 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811911011475}, DOI = {10.1016/j.neuroimage.2011.09.072}, author = {Helbig, HB and Ernst, MO and Ricciardi, E and Pietrini, P and Thielscher, A and Mayer, KM and Schultz, J and Noppeney, U} } @Article { ParkerJonesGGPFALRGPSP2011, title = {Where, When and Why Brain Activation Differs for Bilinguals and Monolinguals during Picture Naming and Reading Aloud}, journal = {Cerebral Cortex}, year = {2012}, month = {4}, volume = {22}, number = {4}, pages = {892-902}, abstract = {Using functional magnetic resonance imaging, we found that when bilinguals named pictures or read words aloud, in their native or nonnative language, activation was higher relative to monolinguals in 5 left hemisphere regions: dorsal precentral gyrus, pars triangularis, pars opercularis, superior temporal gyrus, and planum temporale. We further demonstrate that these areas are sensitive to increasing demands on speech production in monolinguals. This suggests that the advantage of being bilingual comes at the expense of increased work in brain areas that support monolingual word processing. By comparing the effect of bilingualism across a range of tasks, we argue that activation is higher in bilinguals compared with monolinguals because word retrieval is more demanding; articulation of each word is less rehearsed; and speech output needs careful monitoring to avoid errors when competition for word selection occurs between, as well as within, language.}, department = {Research Group Noppeney}, web_url = {http://cercor.oxfordjournals.org/content/22/4/892.full.pdf+html}, DOI = {10.1093/cercor/bhr161}, author = {Parker Jones, O and Green, DW and Grogan, A and Pliatsikas, C and Filippopolitis, K and Ali, N and Lee, HL and Ramsden, S and Gazarian, K and Prejawa, S and Seghier, ML and Price, CJ} } @Inproceedings { EndresAGN2012, title = {Understanding the Semantic Structure of Human fMRI Brain Recordings with Formal Concept Analysis}, year = {2012}, month = {5}, pages = {96-111}, abstract = {We investigate whether semantic information related to object categories can be obtained from human fMRI BOLD responses with Formal Concept Analysis (FCA). While the BOLD response provides only an indirect measure of neural activity on a relatively coarse spatio-temporal scale, it has the advantage that it can be recorded from humans, who can be questioned about their perceptions during the experiment, thereby obviating the need of interpreting animal behavioral responses. Furthermore, the BOLD signal can be recorded from the whole brain simultaneously. In our experiment, a single human subject was scanned while viewing 72 gray-scale pictures of animate and inanimate objects in a target detection task. These pictures comprise the formal objects for FCA. We computed formal attributes by learning a hierarchical Bayesian classifier, which maps BOLD responses onto binary features, and these features onto object labels. The connectivity matrix between the binary features and the object labels can then serve as the formal context. In line with previous reports, FCA revealed a clear dissociation between animate and inanimate objects with the inanimate category also including plants. Furthermore, we found that the inanimate category was subdivided between plants and non-plants when we increased the number of attributes extracted from the BOLD response. FCA also allows for the display of organizational differences between high-level and low-level visual processing areas. We show that subjective familiarity and similarity ratings are strongly correlated with the attribute structure computed from the BOLD signal.}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/2012/ICFCA-2012-Adam.pdf}, department = {Research Group Noppeney}, web_url = {http://www.econ.kuleuven.be/ICFCA/}, editor = {Domenach, F. , D.I. Ignatov, J. Poelmans}, publisher = {Springer}, address = {Berlin, Germany}, booktitle = {Formal Concept Analysis}, event_place = {Leuven, Belgium}, event_name = {10th International Conference on Formal Concept Analysis (ICFCA 2012)}, ISBN = {978-3-642-29891-2}, DOI = {10.1007/978-3-642-29892-9_13}, author = {Endres, A and Adam, R and Giese, MA and Noppeney, U} } @Inbook { Noppeney2011, title = {Kurt Goldstein und Frederik Buytendijk: Der Leib-Begriff in der organismischen Biologie}, year = {2012}, month = {3}, pages = {194-206}, department = {Research Group Noppeney}, web_url = {http://www.utb-shop.de/details.php?p_id=532397}, editor = {Alloa, E. , T. Bedorf, C. Gr{\"u}ny, T. N. Klass}, publisher = {Mohr Siebeck}, address = {T{\"u}bingen, Germany}, booktitle = {Leiblichkeit: Geschichte und Aktualit{\"a}t eines Konzepts}, ISBN = {978-3-8252-3633-5}, author = {Noppeney, U} } @Inbook { 6652, title = {Characterization of Multisensory Integration with fMRI: Experimental Design, Statistical Analysis, and Interpretation}, year = {2012}, month = {1}, pages = {233-252}, abstract = {This chapter reviews the potential and limitations of functional magnetic resonance imaging (fMRI) in characterizing the neural processes underlying multisensory integration. The neural basis of multisensory integration can be characterized from two distinct perspectives. From the perspective of functional specialization, we aim to identify regions where information from different senses converges and/or is integrated. From the perspective of functional integration, we investigate how information from multiple sensory regions is integrated via interactions among brain regions. Combining these two perspectives, this chapter discusses experimental design, analysis approaches, and interpretational limitations of fMRI results. The first section describes univariate statistical analyses of fMRI data and emphasizes the interpretational ambiguities of various statistical criteria that are commonly used for the identification of multisensory integration sites. The second section explores the potential and limitations of multivariate and pattern classifier approaches in multisensory integration. The third section introduces effective connectivity analyses that investigate how multisensory integration emerges from distinct interactions among brain regions. The complementary strengths of data-driven and hypothesis-driven effective connectivity analyses will be discussed. We conclude by emphasizing that the combined potentials of these various analysis approaches may help us to overcome or at least ameliorate the interpretational ambiguities associated with each analysis when applied in isolation.}, department = {Research Group Noppeney}, web_url = {http://www.crcnetbase.com/doi/abs/10.1201/b11092-17}, editor = {Murray, M. M. , M. T. Wallace}, publisher = {CRC Press}, address = {Boca Raton, FL, USA}, series = {Frontiers in Neuroscience}, booktitle = {The neural bases of multisensory processes}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, ISBN = {978-1-439-81217-4}, DOI = {10.1201/b11092-17}, author = {Noppeney, U} } @Poster { RoheN2012_2, title = {Neural audiovisual representations of space in sensory and higher multisensory cortices}, year = {2012}, month = {10}, volume = {42}, number = {463.12}, abstract = {Previous research has demonstrated that human observers locate audiovisual signals in space by averaging auditory (A) and visual (V) spatial signals according to their relative sensory reliabilities (=inverse of variance) (Ernst \& Banks, 2002; Alais \& Burr, 2004). This form of audiovisual integration is optimal in that it provides the most reliable percept.Yet, the neural systems mediating integration of spatial inputs remain unclear. Multisensory integration of spatial signals has previously been related to higher order association areas such as intraparietal sulcus (IPS) as well as the planum temporale (PT; Bonath et al., 2007). In the current fMRI study, we investigated whether and how early sensory (auditory cortex (A1), PT; visual regions V1-V3) and higher association (IPS) areas represent A and V spatial information. Subjects were presented with synchronous audiovisual signals, at spatially congruent or discrepant locations along the azimuth and at two levels of sensory reliability. Hence, the experimental design factorially manipulated: (1) V location, (2) A location, (3) V reliability. Subjects’ task was to localize the A signal. At the behavioral level, the perceived location of the A input was shifted towards the location of the V input depending on the relative A and V reliabilities. Likewise, at the neural level, the spatial location decoded with linear support vector machines from fMRI signals in brain areas along the A and V processing hierarchies was determined by the relative sensory reliabilities. The spatial location decoded from A1/PT was determined primarily by A spatial information with a stronger influence from V spatial information when the V reliability was high. Conversely, the spatial location decoded from visual areas (V1, V2, V3) and IPS was determined primarily by V spatial information with a stronger A influence when the V information was less reliable. In conclusion, our results suggest that the brain represents audiovisual spatial location in qualitative agreement with reliability-weighted multisensory integration at multiple levels of the cortical processing hierarchy.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sfn.org/am2012/}, event_place = {New Orleans, LA, USA}, event_name = {42nd Annual Meeting of the Society for Neuroscience (Neuroscience 2012)}, author = {Rohe, T and Noppeney, U} } @Poster { EndresANG2012_2, title = {Explicit coding in the brain: data-driven semantic analysis of human fMRI BOLD responses with Formal Concept Analysis}, journal = {Frontiers in Computational Neuroscience}, year = {2012}, month = {9}, volume = {Conference Abstract: Bernstein Conference 2012}, pages = {166}, abstract = {Understanding how semantic information is represented in the brain has been an important research focus of neuroscience in the past few years. We showed previously (Endres et al 2010) that Formal Concept Analysis (FCA, (Ganter and Wille 1999)) can reveal interpretable semantic information (e.g. specialization hierarchies, or feature-based representation) from electrophysiological data. Unlike other analysis methods (e.g. hierarchical clustering), FCA does not impose inappropriate structure on the data. FCA is a mathematical formalism compatible with the explicit coding hypothesis (Foldiak, 2009) Here, we investigate whether similar findings can be obtained from fMRI BOLD responses recorded from human subjects. While the BOLD response provides only an indirect measure of neural activity on a much coarser spatio-temporal scale than electrophysiological recordings, it has the advantage that it can be recorded from humans, which can be questioned about their perceptions during the experiment, thereby obviating the need of interpreting animal behavioural responses. Furthermore, the BOLD signal can be recorded from the whole brain simultaneously. In our experiment, a single human subject was scanned while viewing 72 grayscale pictures of animate and inanimate objects in a target detection task (Siemens Trio 3T scanner, GE-EPI, TE=40ms, 38 axial slices, TR=3.08s, 48 sessions, amounting to a total of 10,176 volume images). These pictures comprise the formal objects for FCA. We computed formal attributes by learning a hierarchical Bayesian classifier, which maps BOLD responses onto binary features, and these features onto object labels. The connectivity matrix between the binary features and the object labels can then serve as the formal context. In line with previous reports, FCA revealed a clear dissociation between animate and inanimate objects in a high-level visual area (inferior temporal cortex, IT), with the inanimate category including plants. The inanimate category was subdivided into plants and non-plants when we increased the number of attributes extracted from the fMRI responses. FCA also highlighted organizational differences between the IT and the primary visual cortex, V1. We show that subjective familiarity and similarity ratings are strongly correlated with the attribute structure computed from the fMRI signal.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.frontiersin.org/10.3389/conf.fncom.2012.55.00056/event_abstract}, event_place = {M{\"u}nchen, Germany}, event_name = {Bernstein Conference 2012}, DOI = {10.3389/conf.fncom.2012.55.00056}, author = {Endres, D and Adam, R and Noppeney, U and Giese, MA} } @Poster { RoheN2012_3, title = {Intraparietal sulcus represents audiovisual space}, journal = {Frontiers in Computational Neuroscience}, year = {2012}, month = {9}, volume = {Conference Abstract: Bernstein Conference 2012}, pages = {192-193}, abstract = {Previous research has demonstrated that human observers locate audiovisual (AV) signals in space by averaging auditory (A) and visual (V) spatial signals according to their relative sensory reliabilities (=inverse of variance) (Ernst \& Banks, 2002; Alais \& Burr, 2004). This form of AV integration is optimal in that it provides the most reliable percept. Yet, the neural systems mediating integration of spatial inputs remain unclear. Multisensory integration of spatial signals has previously been related to higher order association areas such as intraparietal sulcus (IPS) as well as early sensory areas like the planum temporale (Bonath et al., 2007). In the current fMRI study, we investigated whether and how early visual (V1-V3) and higher association (IPS) areas represent A and V spatial information given their retinotopic organization. One subject was presented with synchronous audiovisual signals, at spatially congruent or discrepant locations along the azimuth and at two levels of sensory reliability. Hence, the experimental design factorially manipulated: (1) V location, (2) A location, (3) V reliability. The subject’s task was to localize the A signal. Retinotopic maps in visual areas and IPS were measured with standard wedge and ring checkerboard stimuli. At the behavioral level, the perceived location of the A input was shifted towards the location of the V input depending on the relative A and V reliabilities. At the neural level, the cue locations represented in retinotopic maps were decoded by computing a population vector estimate (Pouget et al., 2000) from the voxels’ BOLD responses to the AV cues given the voxels’ preferred visual field coordinate. In early visual areas (V1-V3), the decoded cue locations were determined by the V spatial signal but were independent from the A spatial signal. In IPS, the decoded cue locations were determined by the V and the A spatial signals if relative V reliability was low. In conclusion, our results suggest that the brain represents AV spatial location in IPS in qualitative agreement with reliability-weighted multisensory integration.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.frontiersin.org/10.3389/conf.fncom.2012.55.00054/event_abstract}, event_place = {M{\"u}nchen, Germany}, event_name = {Bernstein Conference 2012}, DOI = {10.3389/conf.fncom.2012.55.00054}, author = {Rohe, T and Noppeney, U} } @Poster { GianiN2012, title = {Awareness related auditory scene analysis: A processing cascade enables a tone pair to be segregated from background and enter awareness}, year = {2012}, month = {8}, number = {MO-76}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.biomag2012.org/}, event_place = {Paris, France}, event_name = {8th International Conference on Biomagnetism (BIOMAG 2012)}, author = {Giani, AS and Noppeney, U} } @Poster { LeeN2012, title = {Leading or Lagging: Temporal prediction errors are expressed in auditory and visual cortices}, year = {2012}, month = {6}, number = {997}, abstract = {Introduction: In our natural environment our brain is exposed to a constant influx of multisensory signals that dynamically evolve at multiple timescales. Statistical regularities are important cues informing the brain whether two sensory signals are generated by a common physical process and should hence be integrated. This fMRI study investigated how the brain detects violations of these statistical regularities induced by the temporal misalignment of the visual and auditory signals. Specifically, we arbitrated between two hypotheses that make opposite predictions: Under the predictive coding framework the brain iteratively optimizes an internal model of its multisensory environment by reducing the error between its predictions and the sensory inputs. An audiovisual misalignment that violates the natural statistical regularities should thus induce a prediction error signal. For visual leading asynchrony, we would expect a prediction error signal in the auditory cortex, because the delayed auditory signal violates the temporal predictions of the 'leading' visual system (vice versa for auditory leading asynchrony) [2,3]. Alternatively, from the perspective of the biased competition model, the misaligned auditory and visual signals compete for processing resources. For visual leading asynchrony, we would expect an increased BOLD-signal in the visual system indexing the higher salience of the leading visual signal which then suppresses the temporally incompatible auditory signal [1]. Methods: 37 subjects participated in this fMRI study (Siemens TimTrio 3T scanner, GE-EPI, TE = 40 ms, 42 axial slices, TR = 3s). They passively perceived audiovisual movies of natural speech, sinewave speech (SWS) and piano music. The audiovisual signals were synchronous, auditory leading (+240ms) or visual leading (-240ms). Hence, the 3 x 3 factorial design manipulated (i) temporal alignment (3 levels) and (ii) stimulus class (3 levels). The activation trials were interleaved with 8s fixation blocks. To allow for random-effects analyses, contrast images (single condition > fixation) for each subject were entered into a 2nd level ANOVA, which modelled the 9 effects in our 3 X 3 design. 1. Using a conjunction null conjunction analysis, we identified differences between auditory and visual leading conditions that are common to speech, SWS and music. 2. We tested for asynchrony effects (i.e. auditory leading > synchronous, visual leading > synchronous) separately for each stimulus class. Results are reported at p<.05 corrected for multiple comparisons at the cluster level using a height threshold of p<.001 uncorrected. Results: 1. Common for all stimulus classes, auditory leading relative to visual leading signals increased activations in bilateral V5/hMT+. In contrast, visual leading relative to auditory leading signals increased activations in bilateral Heschl's gyri (Fig. 1). 2. Auditory leading relative to synchronous AV signals increased activations in the auditory system extending from Heschl's gyrus into posterior superior temporal sulcus/gyrus (STS/STG) bilaterally. Conversely, visual leading relative to synchronous signals increased activations in bilateral occipito-temporal cortices predominantly in V5/hMT+ (Fig. 2).}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/2012/OHBM-2012-Lee.pdf}, department = {Research Group Noppeney}, web_url = {http://ww4.aievolution.com/hbm1201/index.cfm?do=abs.viewAbs\&abs=6296}, event_place = {Beijing, China}, event_name = {18th Annual Meeting of the Organization for Human Brain Mapping (OHBM 2012)}, author = {Lee, HL and Noppemey, U} } @Conference { GianiN2012_2, title = {Towards multisensory awareness: Finding (neuronal) mechanisms that enable the detection and integration of audiovisual stimuli}, year = {2012}, month = {11}, volume = {13}, pages = {8}, abstract = {In daily life our sensory systems continuously receive complex information from different sensory modalities, such as vision, audition or touch. To form unified and coherent percepts this information needs to be integrated across the various senses; a process called multisensory integration. Multisensory information stemming from natural environments, such as market places or busy roads, can be extremely diverse. Moreover limited processing capacities allow only a small subset of the complex sensory information to enter awareness. Hence, two main questions arise: Which are the (neuronal) mechanisms that enable sensory awareness? And is perceptual awareness necessary for multisensory integration to occur? During my PhD I used magnetoencephalography (MEG) and psychophysical measurements, trying to find some answers to these questions, which I will be presenting during this talk.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://www.neuroschool-tuebingen-nena.de/}, event_place = {Schramberg, Germany}, event_name = {13th Conference of the Junior Neuroscientists of T{\"u}bingen (NeNA 2012)}, author = {Giani, A and Noppeney, U} } @Conference { EndresANG2012, title = {Explicit coding in the brain: data-driven semantic analysis of human fMRI BOLD responses with Formal Concept Analysis}, journal = {Perception}, year = {2012}, month = {9}, volume = {41}, number = {ECVP Abstract Supplement}, pages = {67}, abstract = {We investigated whether semantic information about object categories can be obtained from human fMRI BOLD responses with Formal Concept Analysis (FCA), an order-theoretic approach for the analysis of semantic information, such as specialization hierarchies and parts-based codes. Unlike other analysis methods (eg hierarchical clustering), FCA does not impose inappropriate structure on the data. FCA is a mathematical formulation of the explicit coding hypothesis (Foldiak, 2009 Current Biology19R904-R906). A human subject was scanned viewing 72 gray-scale pictures of animate and inanimate objects in a target detection task. To apply FCA, we employ a hierarchical Bayesian classifier, which maps fMRI responses onto binary attributes, and these onto object labels. The connectivity matrix between attributes and labels is the formal context for FCA. FCA revealed a clear dissociation between animate and inanimate objects in a high-level visual area (inferior temporal cortex, IT), with the inanimate category including plants. The inanimate category was subdivided into plants and non-plants when we increased the number of attributes extracted from the fMRI responses. FCA also displayed organizational differences between the IT and the primary visual cortex, V1. We show that familiarity and similarity ratings are strongly correlated with the attributes computed from the fMRI signal.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://www.perceptionweb.com/abstract.cgi?id=v120406}, event_place = {Alghero, Italy}, event_name = {35th European Conference on Visual Perception}, author = {Endres, D and Adam, R and Noppeney, U and Giese, M} } @Conference { AdamN2012, title = {Processing of audiovisual phonological incongruency depends on awareness}, journal = {Seeing and Perceiving}, year = {2012}, month = {6}, day = {21}, volume = {25}, pages = {168}, abstract = {Capacity limitations of attentional resources allow only a fraction of sensory inputs to enter our awareness. Most prominently, in the attentional blink, the observer fails to detect the second of two rapidly successive targets that are presented in a sequence of distractor items. This study investigated whether phonological (in)congruency between visual target letters and spoken letters is modulated by subjects’ awareness. In a visual attentional blink paradigm, subjects were presented with two visual targets (buildings and capital Latin letters, respectively) in a sequence of rapidly presented distractor items. A beep was presented always with T1. We manipulated the presence/absence and phonological congruency of the spoken letter that was presented concurrently with T2. Subjects reported the identity of T1 and T2 and reported the visibility of T2. Behaviorally, subjects correctly identified T2 when it was reported to be either visible or unsure, while performances were below chance level when T2 was reported to be invisible. At the neural level, the anterior cingulate was activated for invisible>unsure>visible T2. In contrast, visible relative to invisible trials increased activation in bilateral cerebellum, pre/post-central gyri extending into parietal sulci and bilateral inferior occipital gyri. Incongruency effects were observed in the left inferior frontal gyrus, caudate nucleus and insula only for visible stimuli. In conclusion, phonological incongruency is processed differently when subjects are aware of the visual stimulus. This indicates that multisensory integration is not automatic but depends on subjects’ cognitive state.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://booksandjournals.brillonline.com/content/10.1163/187847612x647982}, event_place = {Oxford, UK}, event_name = {13th International Multisensory Research Forum (IMRF 2012)}, DOI = {10.1163/187847612X647982}, author = {Adam, R and Noppeney, U} } @Conference { NoppeneyASMLWOLC2012, title = {Different classes of audiovisual correspondences are processed at distinct levels of the cortical hierarchy}, journal = {Seeing and Perceiving}, year = {2012}, month = {6}, day = {20}, volume = {25}, pages = {69}, abstract = {The brain should integrate sensory inputs only when they emanate from a common source and segregate those from different sources. Sensory correspondences are important cues informing the brain whether two sensory inputs are generated by a common event and should hence be integrated. Most prominently, sensory inputs should co-occur in time and space. More complex audiovisual stimuli may also be congruent in terms of semantics (e.g., objects and source sounds) or phonology (e.g., spoken and written words; linked via common linguistic labels). Surprisingly, metaphoric relations (e.g., pitch and height) have also been shown to influence audiovisual integration. The neural mechanisms that mediate these metaphoric congruency effects are only poorly understood. They may be mediated via (i) natural multisensory binding, (ii) common linguistic labels or (iii) semantics. In this talk, we will present a series of studies that investigate whether these different types of audiovisual correspondences are processed by distinct neural systems. Further, we investigate how those systems are employed by metaphoric audiovisual correspondences. Our results demonstrate that different classes of audiovisual correspondences influence multisensory integration at distinct levels of the cortical hierarchy. Spatiotemporal incongruency is detected already at the primary cortical level. Natural (e.g., motion direction) and phonological incongruency influences MSI in areas involved in motion or phonological processing. Critically, metaphoric interactions emerge in neural systems that are shared with natural and semantic incongruency. This activation pattern may reflect the ambivalent nature of metaphoric audiovisual interactions relying on both natural and semantic correspondences.}, department = {Research Group Noppeney}, department2 = {Department B{\"u}lthoff}, talk_type = {Abstract Talk}, web_url = {http://booksandjournals.brillonline.com/content/10.1163/187847612x646901}, event_place = {Oxford, UK}, event_name = {13th International Multisensory Research Forum (IMRF 2012)}, DOI = {10.1163/187847612X646901}, author = {Noppeney, U and Adam, R and Sadaghiani, S and Maier, JX and Lee, HL and Werner, S and Ostwald, D and Lewis, R and Conrad, V} } @Conference { ConradVN2012_2, title = {Interactions between apparent motion rivalry in vision and touch}, journal = {Seeing and Perceiving}, year = {2012}, month = {6}, day = {19}, volume = {25}, pages = {26-27}, abstract = {Introduction In multistable perception, the brain alternates between several perceptual explanations of ambiguous sensory signals. Recent studies have demonstrated crossmodal interactions between ambiguous and unambiguous signals. However it is currently unknown whether multiple bistable processes can interact across the senses. [1, 2] Using the apparent motion quartet in vision and touch, this study investigated whether bistable perceptual processes for vision and touch are independent or influence each other when powerful cues of congruency are provided to facilitate visuotactile integration. [3] Methods When two visual flashes and/or tactile vibration pulses are presented alternately along the two diagonals of the rectangle, subjects’ percept vacillates between vertical and horizontal apparent motion in the visual and/or tactile modalities [4]. Observers were presented with unisensory (visual/tactile), visuotactile spatially congruent and incongruent apparent motion quartets and reported their visual or tactile percepts. Results Congruent stimulation induced pronounced visuotactile interactions as indicated by increased dominance times and \%-bias for the percept already dominant under unisensory stimulation. Yet, the temporal dynamics did not converge for congruent stimulation. It depended also on subjects’ attentional focus and was generally slower for tactile than visual reports. Conclusion Our results support Bayesian approaches to perceptual inference, where the probability of a perceptual interpretation is determined by combining a modality-specific prior with incoming visual and/or tactile evidence. Under congruent stimulation, joint evidence from both senses decelerates the rivalry dynamics by stabilizing the more likely perceptual interpretation. Importantly, the perceptual stabilization was specific to spatiotemporally congruent visuotactile stimulation indicating multisensory rather than cognitive bias mechanisms.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, department3 = {Research Group Ernst}, talk_type = {Abstract Talk}, web_url = {http://booksandjournals.brillonline.com/content/10.1163/187847612x646497}, event_place = {Oxford, UK}, event_name = {13th International Multisensory Research Forum (IMRF 2012)}, DOI = {10.1163/187847612X646497}, author = {Conrad, V and Vitello, MP and Noppeney, U} } @Article { LeeN2011_2, title = {Long-term music training tunes how the brain temporally binds signals from multiple senses}, journal = {Proceedings of the National Academy of Sciences of the United States of America}, year = {2011}, month = {12}, volume = {108}, number = {51}, pages = {E1441-E1450}, abstract = {Practicing a musical instrument is a rich multisensory experience involving the integration of visual, auditory, and tactile inputs with motor responses. This combined psychophysics–fMRI study used the musician's brain to investigate how sensory-motor experience molds temporal binding of auditory and visual signals. Behaviorally, musicians exhibited a narrower temporal integration window than nonmusicians for music but not for speech. At the neural level, musicians showed increased audiovisual asynchrony responses and effective connectivity selectively for music in a superior temporal sulcus-premotor-cerebellar circuitry. Critically, the premotor asynchrony effects predicted musicians’ perceptual sensitivity to audiovisual asynchrony. Our results suggest that piano practicing fine tunes an internal forward model mapping from action plans of piano playing onto visible finger movements and sounds. This internal forward model furnishes more precise estimates of the relative audiovisual timings and hence, stronger prediction error signals specifically for asynchronous music in a premotor-cerebellar circuitry. Our findings show intimate links between action production and audiovisual temporal binding in perception.}, department = {Research Group Noppeney}, department2 = {Department B{\"u}lthoff}, web_url = {http://www.pnas.org/content/108/51/E1441.full.pdf+html}, DOI = {10.1073/pnas.1115267108}, author = {Lee, HL and Noppeney, U} } @Article { LeoRFLD2011, title = {Looming sounds enhance orientation sensitivity for visual stimuli on the same side as such sounds}, journal = {Experimental Brain Research}, year = {2011}, month = {9}, volume = {213}, number = {2-3}, pages = {193-201}, abstract = {Several recent multisensory studies show that sounds can influence visual processing. Some visual judgments can be enhanced for visual stimuli near a sound occurring around the same time. A recent TMS study (Romei et al. 2009) indicates looming sounds might influence visual cortex particularly strongly. But unlike most previous behavioral studies of possible audio–visual exogenous effects, TMS phosphene thresholds rather than judgments of external visual stimuli were measured. Moreover, the visual hemifield assessed relative to the hemifield of the sound was not varied. Here, we compared the impact of looming sounds to receding or “static” sounds, using auditory stimuli adapted from Romei et al. (2009), but now assessing any influence on visual orientation discrimination for Gabor patches (well-known to involve early visual cortex) when appearing in the same hemifield as the sound or on the opposite side. The looming sounds that were effective in Romei et al. (2009) enhanced visual orientation sensitivity (d\(\prime\)) here on the side of the sound, but not for the opposite hemifield. This crossmodal, spatially specific effect was stronger for looming than receding or static sounds. Similarly to Romei et al. (2009), the differential effect for looming sounds was eliminated when using white noise rather than structured sounds. Our new results show that looming structured sounds can specifically benefit visual orientation sensitivity in the hemifield of the sound, even when the sound provides no information about visual orientation itself.}, department = {Research Group Noppeney}, web_url = {http://www.springerlink.com/content/658r3680757h1636/fulltext.pdf}, DOI = {10.1007/s00221-011-2742-8}, author = {Leo, F and Romei, V and Freeman, E and L{\`a}davas, E and Driver, J} } @Article { MeyerN2011, title = {Multisensory integration: From fundamental principles to translational research}, journal = {Experimental Brain Research}, year = {2011}, month = {9}, volume = {213}, number = {2-3}, pages = {163-166}, abstract = {We perceive our natural environment via multiple senses. How does our mind and brain integrate these diverse sensory inputs into a coherent and unified percept? This challenging and exciting question has been the focus of a growing multidisciplinary community of researchers that meet regularly at the annual meeting of the International Multisensory Research Forum (IMRF). The IMRF meeting brings together researchers that investigate multisensory integration at multiple levels of description ranging from neurophysiology to behaviour, with research interests from theory to applications. Traditionally, multisensory research has focused on the characterization of the fundamental principles and constraints that govern multisensory integration. Research has then moved forward to address questions of how multisensory integration emerges during development and may be perturbed in cases of disease or ageing. In the long-term, multisensory research will have direct impact in translational studies investigating the benefits of a multisensory environment for patients that are impaired when presented with information in one sensory modality alone. Obviously, this myriad of research topics can only be addressed by combining findings from a multitude of methods including psychophysics, neurophysiology and non-invasive structural and functional imaging in humans. Further, since its inception multisensory research has constantly gained impetus from computational models. Computational models contribute substantially to the progress made in multisensory research by providing a deeper understanding of the current empirical findings and conversely making predictions that guide future research. Most prominently, the normative Bayesian perspective continues to inspire inquiries into the optimality of multisensory integration across various species. This special issue on multisensory processing has resulted from the IMRF meeting held at Liverpool University in 2010. In accordance with previous procedures, the call for papers was not restricted to meeting attendees but was open to the entire multisensory community. As has been the tradition since the first special IMRF issue, we received a large number of high quality submissions leading to strong competition. Many excellent submissions had to be rejected or transferred to other special issues because of space limitations. Nevertheless, we hope that the collection of manuscripts included in this special issue will provide a rich source of reference for the wider multisensory community. Given the multidisciplinarity of the IMRF community, the submitted manuscripts cover a range of the topics that have briefly been highlighted above. For coarse reference, we have grouped the manuscripts into five sections.}, department = {Research Group Noppeney}, web_url = {http://www.springerlink.com/content/a15607v4313m6tv5/fulltext.pdf}, DOI = {10.1007/s00221-011-2803-z}, author = {Meyer, GF and Noppeney, U} } @Article { LeeN2011, title = {Physical and Perceptual Factors Shape the Neural Mechanisms That Integrate Audiovisual Signals in Speech Comprehension}, journal = {Journal of Neuroscience}, year = {2011}, month = {8}, volume = {31}, number = {31}, pages = {11338-11350}, abstract = {Face-to-face communication challenges the human brain to integrate information from auditory and visual senses with linguistic representations. Yet the role of bottom-up physical (spectrotemporal structure) input and top-down linguistic constraints in shaping the neural mechanisms specialized for integrating audiovisual speech signals are currently unknown. Participants were presented with speech and sinewave speech analogs in visual, auditory, and audiovisual modalities. Before the fMRI study, they were trained to perceive physically identical sinewave speech analogs as speech (SWS-S) or nonspeech (SWS-N). Comparing audiovisual integration (interactions) of speech, SWS-S, and SWS-N revealed a posterior–anterior processing gradient within the left superior temporal sulcus/gyrus (STS/STG): Bilateral posterior STS/STG integrated audiovisual inputs regardless of spectrotemporal structure or speech percept; in left mid-STS, the integration profile was primarily determined by the spectrotemporal structure of the signals; more anterior STS regions discarded spectrotemporal structure and integrated audiovisual signals constrained by stimulus intelligibility and the availability of linguistic representations. In addition to this “ventral” processing stream, a “dorsal” circuitry encompassing posterior STS/STG and left inferior frontal gyrus differentially integrated audiovisual speech and SWS signals. Indeed, dynamic causal modeling and Bayesian model comparison provided strong evidence for a parallel processing structure encompassing a ventral and a dorsal stream with speech intelligibility training enhancing the connectivity between posterior and anterior STS/STG. In conclusion, audiovisual speech comprehension emerges in an interactive process with the integration of auditory and visual signals being progressively constrained by stimulus intelligibility along the STS and spectrotemporal structure in a dorsal fronto-temporal circuitry.}, department = {Research Group Noppeney}, web_url = {http://www.jneurosci.org/content/31/31/11338.full.pdf+html}, DOI = {10.1523/​JNEUROSCI.6510-10.2011}, author = {Lee, HL and Noppeney, U} } @Article { 6717, title = {Auditory-Motor Expertise Alters ''Speech Selectivity'' in Professional Musicians and Actors}, journal = {Cerebral Cortex}, year = {2011}, month = {4}, volume = {21}, number = {4}, pages = {938-948}, abstract = {Several perisylvian brain regions show preferential activation for spoken language above and beyond other complex sounds. These “speech-selective” effects might be driven by regions’ intrinsic biases for processing the acoustical or informational properties of speech. Alternatively, such speech selectivity might emerge through extensive experience in perceiving and producing speech sounds. This functional magnetic resonance imaging (fMRI) study disambiguated such audiomotor expertise from speech selectivity by comparing activation for listening to speech and music in female professional violinists and actors. Audiomotor expertise effects were identified in several right and left superior temporal regions that responded to speech in all participants and music in violinists more than actresses. Regions associated with the acoustic/information content of speech were identified along the entire length of the superior temporal sulci bilaterally where activation was greater for speech than music in all participants. Finally, an effect of performing arts training was identified in bilateral premotor regions commonly activated by finger and mouth movements as well as in right hemisphere “language regions.” These results distinguish the seemingly speech-specific neural responses that can be abolished and even reversed by long-term audiomotor experience.}, department = {Research Group Noppeney}, web_url = {http://cercor.oxfordjournals.org/content/21/4/938.full.pdf+html}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/cercor/bhq166}, author = {Dick, F and Lee, HL and Nusbaum, H and Price, CJ} } @Article { 6741, title = {The Contributions of Transient and Sustained Response Codes to Audiovisual Integration}, journal = {Cerebral Cortex}, year = {2011}, month = {4}, volume = {21}, number = {4}, pages = {920-931}, abstract = {Multisensory events in our natural environment unfold at multiple temporal scales over extended periods of time. This functional magnetic resonance imaging study investigated whether the brain uses transient (onset, offset) or sustained temporal codes to effectively integrate incoming visual and auditory signals within the cortical hierarchy. Subjects were presented with 1) velocity-modulated radial motion, 2) amplitude-modulated sound, or 3) an in phase combination of both in blocks of variable durations to dissociate transient and sustained blood oxygen level–dependent responses. Audiovisual interactions emerged primarily for transient onset and offset responses highlighting the importance of rapid stimulus transitions for multisensory integration. Strikingly, audiovisual interactions for onset and offset transients were dissociable at the functional and anatomical level. Low-level sensory areas integrated audiovisual inputs at stimulus onset in a superadditive fashion to enhance stimulus salience. In contrast, higher order association areas showed subadditive integration profiles at stimulus offset possibly reflecting the formation of higher order representations. In conclusion, multisensory integration emerges at multiple levels of the cortical hierarchy using different temporal codes and integration profiles. From a methodological perspective, these results highlight the limitations of conventional event related or block designs that cannot characterize these rich dynamics of audiovisual integration.}, department = {Research Group Noppeney}, web_url = {http://cercor.oxfordjournals.org/content/21/4/920.full.pdf+html}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/cercor/bhq161}, author = {Werner, S and Noppeney, U} } @Article { 6313, title = {Audiovisual asynchrony detection in human speech}, journal = {Journal of Experimental Psychology: Human Perception and Performance}, year = {2011}, month = {2}, volume = {37}, number = {1}, pages = {245-256}, abstract = {Combining information from the visual and auditory senses can greatly enhance intelligibility of natural speech. Integration of audiovisual speech signals is robust even when temporal offsets are present between the component signals. In the present study, we characterized the temporal integration window for speech and nonspeech stimuli with similar spectrotemporal structure to investigate to what extent humans have adapted to the specific characteristics of natural audiovisual speech. We manipulated spectrotemporal structure of the auditory signal, stimulus length, and task context. Results indicate that the temporal integration window is narrower and more asymmetric for speech than for nonspeech signals. When perceiving audiovisual speech, subjects tolerate visual leading asynchronies, but are nevertheless very sensitive to auditory leading asynchronies that are less likely to occur in natural speech. Thus, speech perception may be fine-tuned to the natural statistics of audiovisual speech, where facial movements always occur before acoustic speech articulation.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Ernst}, department3 = {Research Group Noppeney}, web_url = {http://psycnet.apa.org/psycarticles/2010-17527-001.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1037/a0019952}, author = {Maier, JX and Di Luca, M and Noppeney, U} } @Poster { AdamEGN2011, title = {Semantic relationships in the tool-selective network revealed by formal concept analysis}, year = {2011}, month = {11}, volume = {41}, number = {486.07}, abstract = {How are objects represented in the human brain? This question has previously been addressed using encoding and decoding approaches to neural activity as indexed by the fMRI BOLD response. Specifically, hierarchical clustering based on similarity matrices of brain activation patterns demonstrated that object representations within the inferior temporal gyrus clustered into animate and inanimate categories in line with previous neuropsychological double dissociations. Here, we used Formal Concept Analysis (FCA) (Ganter and Wille 1999) to characterize how the relationship of BOLD activation patterns maps onto the relationship between object stimuli. FCA displays and interprets the relationship of neural object representations via concept lattices (a type of semantic graph). Each concept is defined by a set of formal objects as extent and a set of formal attributes as intent. In our application, the object stimuli were the formal objects and the binarized activations in single voxels the formal attributes. A single subject was scanned while viewing 72 grayscale pictures of animate and inanimate objects in a target detection task (Siemens Trio 3T scanner, GE-EPI, TE=40ms, 38 axial slices, TR=3.08s, 48 sessions, amounting to a total of 10,176 volume images). We modeled the BOLD responses to the stimulus presentations in an event-related fashion with a general linear model, using a separate regressor for each of the 72 stimuli. From the parameter estimate image for each stimulus, the 300 voxels that were most active for all stimuli > fixation were selected within the category-sensitive system as defined by a prior study (including inferior temporal, supramarginal, inferior frontal gyrus). Formal concept analysis was applied to patterns of thresholded and hence binarized voxel activations. In line with previous reports, formal concept analysis revealed a dissociation between animate and inanimate objects with the inanimate category also including plants and vegetables. This study demonstrates the potential strength of FCA for decoding structured relationships in fMRI data.}, department = {Research Group Noppeney}, web_url = {http://www.sfn.org/am2011/}, event_place = {Washington, DC, USA}, event_name = {41st Annual Meeting of the Society for Neuroscience (Neuroscience 2011)}, author = {Adam, R and Endres, A and Giese, MA and Noppeney, U} } @Poster { TsiatsisN2011, title = {Auditory Processing under Steady State Visual Driving}, year = {2011}, month = {10}, volume = {12}, pages = {44}, abstract = {Neuronal oscillations are considered crucial for information processing in the brain as they can potentially regulate information flow and dynamically bind different cortical and non-cortical regions. This MEG study investigated whether the effect of a transient sound was modulated by the phase of oscillations in the visual cortex. To induce steady state oscillations in the visual cortex, we presented subjects with continuous visual signals luminance-modulated at 4Hz or 10Hz. The transient sounds were presented locked to four phases of the periodic visual stimulus (i.e. 0, 1 2, , 3 4). We then investigated whether the effect of sound depends on the phase of the visual steady state activity by testing for the interaction between sound and visual phase. Conversely, we will investigate the effect of the sound processing on the visual steady state processing given the state of the visual cortex. The results from the two experiments (4Hz \& 10Hz) will be combined and compared. Based on recent neurophysiological evidence, we hypothesize that oscillations at different frequencies play distinct functional roles in multisensory integration.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.neuroschool-tuebingen-nena.de/index.php?id=284}, event_place = {Heiligkreuztal, Germany}, event_name = {12th Conference of Junior Neuroscientists of T{\"u}bingen (NeNA 2011)}, author = {Tsiatsis, P and Noppeney, U} } @Poster { LeoN2011, title = {Conditioning influences audio-visual integration by increasing sound saliency}, journal = {i-Perception}, year = {2011}, month = {10}, volume = {2}, number = {8}, pages = {762}, abstract = {We investigated the effect of prior conditioning an auditory stimulus on audiovisual integration in a series of four psychophysical experiments. The experiments factorially manipulated the conditioning procedure (picture vs. monetary conditioning) and multisensory paradigm (2AFC visual detection vs. redundant target paradigm). In the conditioning sessions, subjects were presented with three pure tones (= conditioned stimulus, CS) that were paired with neutral, positive or negative unconditioned stimuli (US, monetary: +50 euro cents, -50 cents, 0 cents; pictures: highly pleasant, unpleasant and neutral IAPS). In a 2AFC visual selective attention paradigm, detection of near-threshold Gabors was improved by concurrent sounds that had previously been paired with a positive (monetary) or negative (picture) outcome relative to neutral sounds. In the redundant target paradigm, sounds previously paired with positive (monetary) or negative (picture) outcomes increased response speed to both auditory and audiovisual targets similarly. Importantly, prior conditioning did not increase the multisensory response facilitation (i.e. (A+V)/2-AV) or the race model violation. Collectively, our results suggest that prior conditioning primarily increases the saliency of the auditory stimulus per se rather than influencing audiovisual integration directly. In turn, conditioned sounds are rendered more potent for increasing response accuracy or speed in detection of visual targets.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs3/index.php/imrf/2011/paper/view/325}, web_url2 = {http://i-perception.perceptionweb.com/journal/I/volume/2/article/ic762}, event_place = {Fukuoka, Japan}, event_name = {12th International Multisensory Research Forum (IMRF 2011)}, author = {Leo, F and Noppeney, U} } @Poster { GianiEBKPN2011, title = {Steady-state responses in MEG demonstrate information integration within but not across the auditory and visual senses}, year = {2011}, month = {10}, volume = {12}, pages = {27}, abstract = {To form a unified percept of our environment, the human brain integrates information within and across the senses. This MEG study investigated interactions within and between sensory modalities using a frequency analysis of steady-{\^A}{\^a}state responses (SSR) to periodic auditory and/or visual inputs. The 3x3 factorial design, manipulated (1) modality (auditory only, visual only and audiovisual) and (2) temporal dynamics (static, dynamic1 and dynamic2). In the static conditions, subjects were presented with (1) visual gratings, luminance modulated at 6Hz and/or (2) pure tones, frequency modulated at 40 Hz. To manipulate perceptual synchrony, we imposed additional slow modulations on the auditory and visual stimuli either at same (0.2 Hz = synchronous) or different frequencies (0.2 Hz vs. 0.7 Hz = asynchronous). This also enabled us to investigate the integration of two dynamic features within one sensory modality (e.g. a pure tone frequency modulated at 40Hz \& amplitude modulated at 0.2Hz) in the dynamic conditions. We reliably identified crossmodulation frequencies when these two stimulus features were modulated at different frequencies. In contrast, no crossmodulation frequencies were identified when information needed to be combined from auditory and visual modalities. The absence of audiovisual crossmodulation frequencies suggests that the previously reported audiovisual interactions in primary sensory areas may mediate low level spatiotemporal coincidence detection that is prominent for stimulus transients but less relevant for sustained SSR responses. In conclusion, our results indicate that information in SSRs is integrated over multiple time scales within but not across sensory modalities at the primary cortical level.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.neuroschool-tuebingen-nena.de/index.php?id=284}, event_place = {Heiligkreuztal, Germany}, event_name = {12th Conference of Junior Neuroscientists of T{\"u}bingen (NeNA 2011)}, author = {Giani, A and Erick, O and Belardinelli, P and Kleiner, M and Preissl, H and Noppeney, U} } @Poster { RoheN2012, title = {The ventriloquist effect depends on audiovisual spatial discrepancy and visual reliability}, year = {2011}, month = {10}, volume = {12}, pages = {39}, abstract = {Humans integrate auditory and visual spatial cues to locate objects. Generally, location judgments are dominated by vision because observers localize an auditory cue close to a visual cue even if they have been instructed to ignore the latter (ventriloquist effect). A recent model of multisensory integration proposes that the ventriloquist effect is governed by two principles: First, spatially discrepant cues are only integrated if the observer infers that both cues stem from one object (principle of causal inference). Second, if the inference results in an assumption that both cues originate from one object the cues are integrated by weighting them according to their relative reliability (principle of Bayes-optimal cue weighting). Thus, the bimodal estimate of the object`s location has a higher reliability than each of the unisensory estimates per se. In order to test this model, 26 subjects were presented with spatial auditory (HRTF-convolved white noise) and visual cues (cloud of dots). The 5x5x5 factorial design manipulated (1) the auditory cue location, (2) the visual cue location and (3) the reliability of the visual cue via the width of the cloud of dots. Subjects were instructed to locate the auditory cue while ignoring the visual cue and to judge the spatial unity of both cues. In line with the principle of causal inference results showed that the ventriloquist effect was weaker and unity judgments were reduced for larger audiovisual discrepancies. In case of small spatial discrepancies the ventriloquist effect was weaker at low levels of visual reliability implying a Bayes-optimal strategy of cue weighting only if a common cause of both cues was assumed. A probabilistic model incorporating the principles of causal inference and Bayes-optimal cue weighting accurately fitted the behavioral data. Overall, the pattern of results suggested that both principles describe important processes governing multisensory integration.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.neuroschool-tuebingen-nena.de/index.php?id=284}, event_place = {Heiligkreuztal, Germany}, event_name = {12th Conference of Junior Neuroscientists of T{\"u}bingen (NeNA 2011)}, author = {Rohe, T and Noppeney, U} } @Poster { BelardinelliOBNP2011, title = {Mapping human brain function: a comparison between Variational Bayes Techniques and LCMV Beamformer}, year = {2011}, month = {6}, number = {665}, abstract = {In the last years several hierarchical Bayesian approaches to the MEG/EEG inverse problem have provided for a relevant contribution to the field of MEG/EEG source localization (Friston et al., 2008b; Wipf et al., 2010). While several methods show applicability under specific conditions, none is optimal without prior information. Meaningful results are bound to previously acquired information. In this work we used simulated MEG data to compare three Variational Bayes reconstruction algorithms implemented within the SPM software preprocessing framework (available from http://www.fil.ion.ucl.ac.uk/spm/): two approaches involving the search for optimal mixtures of anatomically defined priors (Greedy Search (GS) and Automatic Relevance Determination (ARD)) (Friston et al., 2008a) and a third approach using a single empirical prior based on the well established LCMV Beamformer technique (Van Veen et al., 1997), that we denominated Empirical Bayes Beamformer (EBB). Our parameters of interest were: 1. Number of simulated dipoles (1 to 3), 2. Relative position between dipoles (bilaterally symmetric versus random locations) 3. Dipole time-course correlation level (high/low). Each parameter configuration set was tested with 5 levels of SNR (from -30 to +10 dB) and 50 dipole position sets.}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/2011/HBM-2011-Noppeney.pdf}, department = {Research Group Noppeney}, web_url = {http://www.humanbrainmapping.org/i4a/pages/index.cfm?pageID=3419}, event_place = {Qu{\'e}bec City, Canada}, event_name = {17th Annual Meeting of the Organization for Human Brain Mapping (HBM 2011)}, author = {Belardinelli, P and Ortiz, E and Barnes, G and Noppeney, U and Preissl, H} } @Poster { GianiOBKPN2011, title = {Using steady state responses in MEG to study information integration within and across the senses}, year = {2011}, month = {6}, number = {1028}, abstract = {How does the brain integrate information within and across sensory modalities to form a unified percept? This question has previously been addressed using transient stimuli, analyzed in the time domain. Alternatively, sensory interactions can be investigated using frequency analyses of steady state responses (SSRs). SSRs are elicited by periodic sensory stimulation (such as frequency modulated tones). In the frequency domain, 'true' signal integration is reflected by non-linear crossmodulation terms (i.e. the sums and differences of the individual SSR frequencies). In addition, two signals may modulate the amplitude of the fundamental and harmonic frequencies of one another. Using visual (V) and auditory (A) SSRs, we investigated whether A and V signals are truly integrated as indexed by crossmodulation terms or simply modulate the expression of each other's dominant frequencies. To manipulate perceptual synchrony, we imposed additional slow modulations on the auditory and visual SSRs either at same or different frequencies. This also enabled us to investigate the integration of two dynamic features within one sensory modality.}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/2011/HBM-2011-Giani.pdf}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.humanbrainmapping.org/i4a/pages/index.cfm?pageID=3419}, event_place = {Qu{\'e}bec City, Canada}, event_name = {17th Annual Meeting of the Organization for Human Brain Mapping (HBM 2011)}, author = {Giani, AS and Ortiz, EB and Belardinelli, P and Kleiner, M and Preissl, H and Noppeney, U} } @Conference { LeoN2011_2, title = {Monetary conditioning influences audio-visual integration by increasing sound saliency}, year = {2011}, month = {11}, day = {18}, pages = {7}, abstract = {We investigated the effect of prior conditioning an auditory stimulus on audiovisual integration in a series of three psychophysical experiments. The three experiments employed the same acquisition phase (monetary conditioning) but different multisensory paradigms (Exp1: 2AFC visual detection; Exp2: redundant target paradigm; Exp3: redundant target paradigm using near threshold visual and auditory stimuli). In the acquisition phase, subjects were presented with three pure tones (= conditioned stimuli, CSs) that were paired with positive, negative or neutral unconditioned stimuli (USs, monetary: +50 euro cents, -50 cents, 0 cents). In a 2AFC visual selective attention paradigm, detection of near-threshold Gabors was improved by concurrent sounds that had previously been paired with a positive outcome relative to neutral sounds. Taken together, redundant target paradigms results showed that sounds previously paired with positive or negative outcomes increased response speed to both auditory and audiovisual targets similarly. Importantly, prior conditioning did not increase the multisensory response facilitation (i.e. (A+V)/2-AV) or the race model violation. Collectively, our results suggest that prior conditioning primarily increases the saliency of the auditory stimulus per se rather than influencing audiovisual integration directly. In turn, conditioned sounds are rendered more potent for increasing response accuracy or speed in detection of visual targets.}, department = {Research Group Noppeney}, department2 = {Department B{\"u}lthoff}, talk_type = {Abstract Talk}, web_url = {http://www.sinp-web.org/site/}, institution = {Max Planck Institute for Biological Cybernetics}, event_place = {Bologna, Italy}, event_name = {Riunione Autunnale di Societ{\`a} Italiana di Neuropsicologia (SINP 2011)}, author = {Leo, F and Noppeney, U} } @Conference { SerinoCLFL2011, title = {Extended peripersonal space in professional fencers}, year = {2011}, month = {10}, abstract = {A brief experience of using a tool to act upon far, unreachable, objects quickly modifies the action space, extending transiently the limits of the Peripersonal Space (PPS), i.e., a limited space surrounding the body where stimuli from different sensory modalities are integrated. Here we investigated whether a possible long-term extension of the PPS could be shown in professional fencers, who train everyday with their weapon to “invade” another person’s PPS and “defend” their own PPS. Subjects performed an audio tactile interaction task to assess the PPS around their right hand, either while holding a short handle or their weapon. While holding the handle, the perception of a tactile stimulus at the hand was affected by a sound only when this was perceived near the hand and not in the far space. Conversely, when professional fencers held their weapon, far auditory stimuli interacted with tactile stimuli at the hand, thus suggesting that the boundaries of the multisensory PPS shifted to the tip of the weapon. Preliminary results suggest that the extension effect on the PPS varies between foil and sabre users, due to the different functional use of these two kinds of weapon.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs3/index.php/imrf/2011/paper/view/241}, event_place = {Toulouse, France}, event_name = {12th International Multisensory Research Forum (IMRF 2011)}, author = {Serino, A and Canzoneri, E and Leo, F and Furetta, C and L{\`a}davas, E} } @Article { 6511, title = {Prior auditory information shapes visual category-selectivity in ventral occipito-temporal cortex}, journal = {NeuroImage}, year = {2010}, month = {10}, volume = {52}, number = {4}, pages = {1592-1602}, abstract = {Objects in our natural environment generate signals in multiple sensory modalities. This fMRI study investigated the influence of prior task-irrelevant auditory information on visually-evoked category-selective activations in the ventral occipito-temporal cortex. Subjects categorized pictures as landmarks or animal faces, while ignoring the preceding congruent or incongruent sound. Behaviorally, subjects responded slower to incongruent than congruent stimuli. At the neural level, the lateral and medial prefrontal cortices showed increased activations for incongruent relative to congruent stimuli consistent with their role in response selection. In contrast, the parahippocampal gyri combined visual and auditory information additively: activation was greater for visual landmarks than animal faces and landmark-related sounds than animal vocalizations resulting in increased parahippocampal selectivity for congruent audiovisual landmarks. Effective connectivity analyses showed that this amplification of visual lan dmark-selectivity was mediated by increased negative coupling of the parahippocampal gyrus with the superior temporal sulcus for congruent stimuli. Thus, task-irrelevant auditory information influences visual object categorization at two stages. In the ventral occipito-temporal cortex auditory and visual category information are combined additively to sharpen visual category-selective responses. In the left inferior frontal sulcus, as indexed by a significant incongruency effect, visual and auditory category information are integrated interactively for response selection.}, department = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science?_ob=MImg\&_imagekey=B6WNP-501FPKM-5-N\&_cdi=6968\&_user=29041\&_pii=S1053811910007056\&_orig=search\&_coverDate=05\%2F07\%2F2010\&_sk=999999999\&view=c\&wchp=dGLbVtz-zSkWb\&md5=9500841a0f1e7}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1016/j.neuroimage.2010.05.002}, author = {Adam, R and Noppeney, U} } @Article { 6780, title = {Audiovisual interactions in binocular rivalry}, journal = {Journal of Vision}, year = {2010}, month = {8}, volume = {10}, number = {10:27}, pages = {1-15}, abstract = {When the two eyes are presented with dissimilar images, human observers report alternating percepts—a phenomenon coined binocular rivalry. These perceptual fluctuations reflect competition between the two visual inputs both at monocular and binocular processing stages. Here we investigated the influence of auditory stimulation on the temporal dynamics of binocular rivalry. In three psychophysics experiments, we investigated whether sounds that provide directionally congruent, incongruent, or non-motion information modulate the dominance periods of rivaling visual motion percepts. Visual stimuli were dichoptically presented random-dot kinematograms (RDKs) at different levels of motion coherence. The results show that directional motion sounds rather than auditory input per se influenced the temporal dynamics of binocular rivalry. In all experiments, motion sounds prolonged the dominance periods of the directionally congruent visual motion percept. In contrast, motion sounds abbreviated the suppression periods of the directionally congruent visual motion percepts only when they competed with directionally incongruent percepts. Therefore, analogous to visual contextual effects, auditory motion interacted primarily with consciously perceived visual input rather than visual input suppressed from awareness. Our findings suggest that auditory modulation of perceptual dominance times might be established in a top-down fashion by means of feedback mechanisms.}, department = {Department B{\"u}lthoff}, department2 = {Department Logothetis}, department3 = {Research Group Noppeney}, web_url = {http://www.journalofvision.org/content/10/10/27.full.pdf+html}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1167/10.10.27}, author = {Conrad, V and Bartels, A and Kleiner, M and Noppeney, U} } @Article { 6117, title = {Superadditive Responses in Superior Temporal Sulcus Predict Audiovisual Benefits in Object Categorization}, journal = {Cerebral Cortex}, year = {2010}, month = {8}, volume = {20}, number = {8}, pages = {1829-1842}, abstract = {Merging information from multiple senses provides a more reliable percept of our environment. Yet, little is known about where and how various sensory features are combined within the cortical hierarchy. Combining functional magnetic resonance imaging and psychophysics, we investigated the neural mechanisms underlying integration of audiovisual object features. Subjects categorized or passively perceived audiovisual object stimuli with the informativeness (i.e., degradation) of the auditory and visual modalities being manipulated factorially. Controlling for low-level integration processes, we show higher level audiovisual integration selectively in the superior temporal sulci (STS) bilaterally. The multisensory interactions were primarily subadditive and even suppressive for intact stimuli but turned into additive effects for degraded stimuli. Consistent with the inverse effectiveness principle, auditory and visual informativeness determine the profile of audiovisual integration in STS similarly to the influ ence of physical stimulus intensity in the superior colliculus. Importantly, when holding stimulus degradation constant, subjects’ audiovisual behavioral benefit predicts their multisensory integration profile in STS: only subjects that benefit from multisensory integration exhibit superadditive interactions, while those that do not benefit show suppressive interactions. In conclusion, superadditive and subadditive integration profiles in STS are functionally relevant and related to behavioral indices of multisensory integration with superadditive interactions mediating successful audiovisual object categorization.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/Werner2009_SuperadditiveResponsesInSTSpredictAudiovisualBenefits_6117[0].pdf}, department = {Research Group Noppeney}, web_url = {http://cercor.oxfordjournals.org/cgi/reprint/bhp248v1}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/cercor/bhp248}, author = {Werner, S and Noppeney, U} } @Article { 6410, title = {Developmental dyslexia in Chinese and English: Dissociating the effect of dyslexia from language differences}, journal = {Brain}, year = {2010}, month = {5}, volume = {133}, number = {6}, pages = {1694-1706}, abstract = {Previous neuroimaging studies have suggested that developmental dyslexia has a different neural basis in Chinese and English populations because of known differences in the processing demands of the Chinese and English writing systems. Here, using functional magnetic resonance imaging, we provide the first direct statistically based investigation into how the effect of dyslexia on brain activation is influenced by the Chinese and English writing systems. Brain activation for semantic decisions on written words was compared in English dyslexics, Chinese dyslexics, English normal readers and Chinese normal readers, while controlling for all other experimental parameters. By investigating the effects of dyslexia and language in one study, we show common activation in Chinese and English dyslexics despite different activation in Chinese versus English normal readers. The effect of dyslexia in both languages was observed as less than normal activation in the left angular gyrus and in left middle frontal, posterior temporal and occipitotemporal regions. Differences in Chinese and English normal reading were observed as increased activation for Chinese relative to English in the left inferior frontal sulcus; and increased activation for English relative to Chinese in the left posterior superior temporal sulcus. These cultural differences were not observed in dyslexics who activated both left inferior frontal sulcus and left posterior superior temporal sulcus, consistent with the use of culturally independent strategies when reading is less efficient. By dissociating the effect of dyslexia from differences in Chinese and English normal reading, our results reconcile brain activation results with a substantial body of behavioural studies showing commonalities in the cognitive manifestation of dyslexia in Chinese and English populations. They also demonstrate the influence of cognitive ability and learning environment on a common neural system for reading.}, department = {Research Group Noppeney}, web_url = {http://brain.oxfordjournals.org/content/133/6/1694.full.pdf+html}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/brain/awq106}, author = {Hu, W and Lee, HL and Zhang, Q and Liu, T and Shakeshaft, C and Tae, T and Mohamed, S and David, G and Yang, Y and Price, CJ} } @Article { 6593, title = {Perceptual Decisions Formed by Accumulation of Audiovisual Evidence in Prefrontal Cortex}, journal = {Journal of Neuroscience}, year = {2010}, month = {5}, volume = {30}, number = {21}, pages = {7434-7446}, abstract = {To form perceptual decisions in our multisensory environment, the brain needs to integrate sensory information derived from a common source and segregate information emanating from different sources. Combining fMRI and psychophysics in humans, we investigated how the brain accumulates sensory evidence about a visual source in the context of congruent or conflicting auditory information. In a visual selective attention paradigm, subjects (12 females, 7 males) categorized video clips while ignoring concurrent congruent or incongruent soundtracks. Visual and auditory information were reliable or unreliable. Our behavioral data accorded with accumulator models of perceptual decision making, where sensory information is integrated over time until a criterion amount of information is obtained. Behaviorally, subjects exhibited audiovisual incongruency effects that increased with the variance of the visual and the reliability of the interfering auditory input. At the neural level, only the left inferior frontal sulcus (IFS) showed an ''audiovisual-accumulator'' profile consistent with the observed reaction time pattern. By contrast, responses in the right fusiform were amplified by incongruent auditory input regardless of sensory reliability. Dynamic causal modeling showed that these incongruency effects were mediated via connections from auditory cortex. Further, while the fusiform interacted with IFS in an excitatory recurrent loop that was strengthened for unreliable task-relevant visual input, the IFS did not amplify and even inhibited superior temporal activations for unreliable auditory input. To form decisions that guide behavioral responses, the IFS may accumulate audiovisual evidence by dynamically weighting its connectivity to auditory and visual regions according to sensory reliability and decisional relevance.}, department = {Research Group Noppeney}, web_url = {http://www.jneurosci.org/cgi/reprint/30/21/7434}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1523/JNEUROSCI.0455-10.2010}, author = {Noppeney, U and Ostwald, D and Werner, S} } @Article { 6238, title = {Distinct Functional Contributions of Primary Sensory and Association Areas to Audiovisual Integration in Object Categorization}, journal = {Journal of Neuroscience}, year = {2010}, month = {2}, volume = {30}, number = {7}, pages = {2662-2675}, abstract = {Multisensory interactions have been demonstrated in a distributed neural system encompassing primary sensory and higher-order association areas. However, their distinct functional roles in multisensory integration remain unclear. This fMRI study dissociated the functional contributions of three cortical levels to multisensory integration in object categorization. Subjects actively categorized or passively perceived noisy auditory and visual signals emanating from everyday actions with objects. The experiment included two 2x2 factorial designs that manipulated either (i) the presence/absence or (ii) the informativeness of the sensory inputs. These experimental manipulations revealed three patterns of audiovisual interactions. (1) In primary auditory cortices (PAC), a concurrent visual input increased the stimulus salience by amplifying the auditory response irrespective of task-context. Effective connectivity analyses demonstrated that this automatic response amplification is mediated via both, direct and indi rect (via STS) connectivity to visual cortices. (2) In superior temporal (STS) and intraparietal (IPS) sulci, audiovisual interactions sustained the integration of higher-order object features and predicted subjects’ audiovisual benefits in object categorization. (3) In the left ventrolateral prefrontal cortex (vlPFC), explicit semantic categorization resulted in suppressive audiovisual interactions as an index for multisensory facilitation of semantic retrieval and response selection. In conclusion, multisensory integration emerges at multiple processing stages within the cortical hierarchy. The distinct profiles of audiovisual interactions dissociate audiovisual salience effects in PAC, formation of object representations in STS/IPS and audiovisual facilitation of semantic categorization in vlPFC. Furthermore, in STS/IPS, the profiles of audiovisual interactions were behaviorally relevant and predicted subjects’ multisensory benefits in performance accuracy.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/Werner2010_JoNsc_6238[0].pdf}, department = {Research Group Noppeney}, web_url = {http://www.jneurosci.org/cgi/reprint/30/7/2662}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1523/JNEUROSCI.5091-09.2010}, author = {Werner, S and Noppeney, U} } @Article { 5794, title = {The Role of Right and Left Parietal Lobes in the Conceptual Processing of Numbers}, journal = {Journal of Cognitive Neuroscience}, year = {2010}, month = {2}, volume = {22}, number = {2}, pages = {331-346}, abstract = {Neuropsychological and functional imaging studies have associated the conceptual processing of numbers with bilateral parietal regions (including intraparietal sulcus). However, the processes driving these effects remain unclear because both left and right posterior parietal regions are activated by many other conceptual, perceptual, attention, and response-selection processes. To dissociate parietal activation that is number-selective from parietal activation related to other stimulus or response-selection processes, we used fMRI to compare numbers and object names during exactly the same conceptual and perceptual tasks while factoring out activations correlating with response times. We found that right parietal activation was higher for conceptual decisions on numbers relative to the same tasks on object names, even when response time effects were fully factored out. In contrast, left parietal activation for numbers was equally involved in conceptual processing of object names. We suggest that left parietal activation for numbers reflects a range of processes, including the retrieval of learnt facts that are also involved in conceptual decisions on object names. In contrast, number selectivity in right parietal cortex reflects processes that are more involved in conceptual decisions on numbers than object names. Our results generate a new set of hypotheses that have implications for the design of future behavioral and functional imaging studies of patients with left and right parietal damage.}, department = {Research Group Noppeney}, web_url = {http://www.mitpressjournals.org/doi/pdf/10.1162/jocn.2009.21246}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1162/jocn.2009.21246}, author = {Cappelletti, M and Lee, HL and Freeman, E and Price, CJ} } @Inproceedings { HuszarNL2010, title = {Mind Reading by Machine Learning: A doubly Bayesian Method for Inferring Mental Representations}, year = {2010}, month = {8}, pages = {2810-2815}, abstract = {A central challenge in cognitive science is to measure and quantify the mental representations humans develop \{ in other words, to `read' subject's minds. In order to elimi- nate potential biases in reporting mental contents due to verbal elaboration, subjects' responses in experiments are often limited to binary decisions or discrete choices that do not require conscious re ection upon their mental contents. However, it is unclear what such impoverished data can tell us about the potential richness and dy- namics of subjects' mental representations. To address this problem, we used ideal observer models that for- malise choice behaviour as (quasi-)Bayes-optimal, given subjects' representations in long-term memory, acquired through prior learning, and the stimuli currently avail- able to them. Bayesian inversion of such ideal observer models allowed us to infer subjects' mental representation from their choice behaviour in a variety of psychophysical tasks. The inferred mental representations also allowed us to predict future choices of subjects with reasonable accuracy, even in tasks that were di erent from those in which the representations were estimated. These results demonstrate a signi cant potential in standard binary decision tasks to recover detailed information about sub- jects' mental representations.}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/2011/COGSCI-2010-Huszar.pdf}, department = {Research Group Noppeney}, web_url = {http://www.cognitivesciencesociety.org/conference2010/index.html}, editor = {Ohlsson, S. , R. Catrambone}, publisher = {Cognitive Science Society}, address = {Austin, TX, USA}, booktitle = {Cognition in Flux}, event_place = {Portland, OR, USA}, event_name = {32nd Annual Conference of the Cognitive Science Society (COGSCI 2010)}, author = {Huszar, F and Noppeney, U and Lengyel, M} } @Poster { 7078, title = {Investigating the effect of IPS TMS-stimulation on auditory and visual processing: A TMS-fMRI Study}, year = {2010}, month = {10}, volume = {11}, number = {3}, pages = {22}, abstract = {Recent studies show that the activity in the visual cortex can be decreased by simultaneous auditory input, and vice versa in the auditory cortex. It remains unclear whether these modulations are mediated via low-level interactions between sensory cortices or top-down effects from higher association cortices. The bidirectional anatomical connectivity from the intraparietal sulcus (IPS) to visual and auditory areas renders it ideal for integrating inputs from multiple senses. Here, we aim to determine how applying TMS on the right IPS influences the activation in the visual cortex during (1) visual stimulation (visual induced activations) and (2) auditory stimulation (auditory induced deactivations).}, department = {Research Group Noppeney}, web_url = {http://www.neuroschool-tuebingen-nena.de/fileadmin/user_upload/Dokumente/neuroscience/AbstractbookNeNa2010u.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Heiligkreuztal, Germany}, event_name = {11th Conference of Junior Neuroscientists of T{\"u}bingen (NeNa 2010)}, language = {en}, author = {Leit{\~a}o, J} } @Poster { 7077, title = {Audiovisual Integration in Motion Discrimination: A multivariate decoding study}, year = {2010}, month = {6}, volume = {16}, number = {1587 WTh-AM}, pages = {160}, department = {Research Group Noppeney}, web_url = {http://www.humanbrainmapping.org/files/2010MeetingFiles/OHBM\%202010\%20Abstract\%20Book.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Barcelona, Spain}, event_name = {16th Annual Meeting of the Organisation for Human Brain Mapping (HBM 2010)}, language = {en}, author = {Tuennerhoff, J and Noppeney, U} } @Poster { 6615, title = {Audiovisual synchrony perception of speech and non-speech signals in musicians and non-musicians}, year = {2010}, month = {6}, volume = {16}, number = {1580 WTh-PM}, pages = {212}, department = {Research Group Noppeney}, web_url = {http://www.humanbrainmapping.org/files/2010MeetingFiles/OHBM\%202010\%20Abstract\%20Book.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Barcelona, Spain}, event_name = {16th Annual Meeting of the Organisation for Human Brain Mapping (HBM 2010)}, language = {en}, author = {Lee, HL and Noppeney, U} } @Poster { 6662, title = {Effective Connectivity in Multisensory integration: Insights from functional imaging in humans}, year = {2010}, month = {6}, volume = {11}, number = {126}, abstract = {Multisensory interactions emerge in a distributed neural system encompassing primary sensory and higher-order association areas. Multiple functional brain architectures have been proposed to mediate multisensory interactions in low-level auditory regions including feedforward thalamocortical, direct connections between sensory areas and feedback from higher-order association areas such as IPS or STS. We will review the potential and limitations of combining functional imaging and effective connectivity analyses for characterizing functional architectures of multisensory integration. In a series of three audiovisual integration studies, we combined dynamic causal modeling and Bayesian Model comparison to arbitrate between neural models where crossmodal effects are mediated via ‘direct‘ V1-A1 connectivity, ‘indirect‘ feedback connectivity from STS or both mechanisms. The first study manipulated the presence/absence of auditory and visual inputs and demonstrated that low level audiovisual salience effects are mediated via both direct and indirect mechanisms of audiovisual integration. The second study showed that audiovisual synchrony effects in low-level sensory areas are mediated primarily via direct connectivity. The third study demonstrated that semantic audiovisual (in)congruency effects in higher order visual object areas are elicited by direct influences from auditory areas rather than top-down effects from prefrontal cortices. We conclude by critically reviewing interpretational ambiguities and pitfalls of Dynamic Causal Modelling results based on fMRI data in humans.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs2/index.php/imrf/2010/paper/view/126}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Liverpool, UK}, event_name = {11th International Multisensory Research Forum (IMRF 2010)}, language = {en}, author = {Noppeney, U and Werner, S and Ostwald, D and Lewis, R} } @Poster { 7076, title = {Investigating the effect of IPS TMS-stimulation on auditory and visual processing: A TMS-fMRI Study}, year = {2010}, month = {6}, volume = {16}, number = {10 MT-PM}, pages = {109}, department = {Department MRZ}, department2 = {Research Group Noppeney}, web_url = {http://www.humanbrainmapping.org/files/2010MeetingFiles/OHBM\%202010\%20Abstract\%20Book.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Barcelona, Spain}, event_name = {16th Annual Meeting of the Organisation for Human Brain Mapping (HBM 2010)}, language = {en}, author = {Leit{\~a}o, J and Thielscher, A and Werner, S and Pohmann, R and Noppeney, U} } @Thesis { 6807, title = {The Neural Correlates and Mechanisms Mediating the Integration of Auditory and Visual Information in the Human Brain}, year = {2010}, month = {2}, abstract = {To respond more quickly to events in natural environments the human brain merges information from multiple senses into a more reliable percept. Multisensory integration processes have been demonstrated in a distributed neural system encompassing sensory-specific, higher association and prefrontal cortices. Using fMRI and psychophysical methods this dissertation investigates the functional similarities, differences and constraints that govern the integration of auditory and visual information in different regions of the human cerebral cortex. Characterizing their temporal response codes, effective connectivity patterns and underlying computations for combining multisensory inputs, this work provides evidence for the integration of specific types of information at 3 functionally specialized processing stages. At the first stage, multisensory interactions in sensory-specific regions indicate the same sensory source by integrating spatiotemporally aligned auditory and visual inputs to enhance stimulus detection. At the second stage, multisensory interactions in higher association regions integrate complex environmental features into higher order representations, forming a unified percept and mediating multisensory benefits in object recognition. At the third stage, multisensory interactions in the prefrontal cortex mediate response selection processes based on perceptual information from auditory and visual modalities with multisensory facilitations of reaction time. This dissertation constitutes the first systematic attempt to dissociate the contributions of sensory-specific, higher association and prefrontal areas to audiovisual integration in the human brain.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/SW_ThesisSynopsis_[0].pdf}, department = {Research Group Noppeney}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, institution = {Eberhard-Karls-Universit{\"a}t, T{\"u}bingen, Germany}, type = {PhD}, language = {en}, author = {Werner, S} } @Conference { Leitao2010, title = {TMS des intra-parietalen Sulcus vermindert die reziproke Deaktivierung visueller und auditorischer Kortices: eine simultane fMRT-/TMS-Studie}, year = {2010}, month = {6}, volume = {36}, pages = {17}, abstract = {Um unsere Umwelt verl{\"a}sslich wahrnehmen zu k{\"o}nnen, kombiniert das Gehirn {\"u}ber verschiedene Sinne vermittelte Information. Ein im menschlichen Gehirn beobachtetes Ph{\"a}nomen ist die reziproke Deaktivierung unisensorischer Regionen durch Stimulation in einer anderen Sinnesmodalit{\"a}t. So f{\"u}hrt z.B. visuelle Stimulation zu einer Deaktivierung im auditorischen Kortex and auditorische Stimulation zu einer Deaktivierung im visuellen Kortex. Bisher sind jedoch weder die genauen Mechanismen, die f{\"u}r solche Deaktivierungen verantwortlich sind, noch deren funktionellen Relevanz bekannt. Viele Studien haben gezeigt, dass der multisensorische intraparietale Sulcus (IPS) die Aktivit{\"a}t in sensorischen Regionen modulieren kann. In dieser simultan kombinierten fMRT-/TMS-Studie wurden Probanden visuelle oder auditorische looming-receding Stimuli pr{\"a}sentiert. Um die Aufmerksamkeit der Probanden zu kontrollieren, hatten diese die Aufgabe, seltene auditorische oder visuelle Zielreize zu entdecken. In der H{\"a}lfte der fMRT-Durchg{\"a}nge, stimulierten wir den rechten IPS mit transkranieller Magnetstimulation (TMS), um dadurch den Einfluss dieser Region auf die erw{\"a}hnten reziproken Deaktivierungen unisensorischer Regionen zu untersuchen. Dabei zeigte sich, dass die Deaktivierung im visuellen Kortex w{\"a}hrend auditorischer Stimulation reduziert wurde, wenn der IPS stimuliert wurde. Diese Daten legen nahe, dass der IPS das reziproke Deaktivierungsmuster beeinflusst. Daher scheinen auditorische und visuelle Regionen auch indirekt {\"u}ber den IPS zu interagieren.}, department = {Research Group Noppeney}, web_url = {http://tagung.dgpa.de/fileadmin/files/ABSTRACTBAND_web.pdf}, institution = {Max Planck Institute for Biological Cybernetics}, event_place = {Greifswald, Germany}, event_name = {36. Tagung ''Psychologie und Gehirn''}, author = {Leit{\~a}o, J} } @Article { 5775, title = {The sensory-motor theory of semantics: Evidence from functional imaging}, journal = {Language and Cognition}, year = {2009}, month = {10}, volume = {1}, number = {2}, pages = {249-276}, abstract = {This review discusses the contributions of functional imaging (fMRI/PET) to our understanding of how semantic concepts are represented and processed in the human brain. The sensory-motor theory of semantic memory suggests that semantic processing relies on reactivation of sensory-motor representations that were involved in perception and action. More specifically, it attributes an apparent category-specific (e.g. tool vs. animals) organization of semantics to anatomical segregation for different semantic features (e.g. action vs. visual). Within this framework, we will review functional imaging evidence that semantic processing of tools and actions may rely on activations within the visuo-motor system.}, department = {Research Group Noppeney}, web_url = {http://www.reference-global.com/doi/pdf/10.1515/LANGCOG.2009.012}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1515/LANGCOG.2009.012}, author = {Noppeney, U} } @Article { 5912, title = {Natural, Metaphoric, and Linguistic Auditory Direction Signals Have Distinct Influences on Visual Motion Processing}, journal = {Journal of Neuroscience}, year = {2009}, month = {5}, volume = {29}, number = {20}, pages = {6490-6499}, abstract = {To interact with our dynamic environment, the brain merges motion information from auditory and visual senses. However, not only ''natural'' auditory MOTION, but also ''metaphoric'' de/ascending PITCH and SPEECH (e.g., ''left/right''), influence the visual motion percept. Here, we systematically investigate whether these three classes of direction signals influence visual motion perception through shared or distinct neural mechanisms. In a visual-selective attention paradigm, subjects discriminated the direction of visual motion at several levels of reliability, with an irrelevant auditory stimulus being congruent, absent, or incongruent. Although the natural, metaphoric, and linguistic auditory signals were equally long and adjusted to induce a comparable directional bias on the motion percept, they influenced visual motion processing at different levels of the cortical hierarchy. A significant audiovisual interaction was revealed for MOTION in left human motion complex (hMT+/V5+) and for SPEECH in right intraparietal sulcus. In fact, the audiovisual interaction gradually decreased in left hMT+/V5+ for MOTION > PITCH > SPEECH and in right intraparietal sulcus for SPEECH > PITCH > MOTION. In conclusion, natural motion signals are integrated in audiovisual motion areas, whereas the influence of culturally learnt signals emerges primarily in higher-level convergence regions.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.jneurosci.org/cgi/reprint/29/20/6490}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1523/JNEUROSCI.5437-08.2009}, author = {Sadaghiani, S and Maier, JX and Noppeney, U} } @Inbook { 5229, title = {The sensory-motor theory of semantics: Evidence from functional imaging}, year = {2009}, month = {1}, pages = {177-210}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.peterlang.com/index.cfm?vID=57630\&amp;vLang=D\&amp;vHR=1\&amp;vUR=2\&amp;vUUR=3}, editor = {Fuchs, S. , H. Loevenbruck, D. Pape, P. Perrier}, publisher = {Lang}, address = {Frankfurt (Main), Germany}, booktitle = {Some Aspects of Speech and the Brain}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, ISBN = {978-3-631-57630-4}, author = {Noppeney, U} } @Poster { 5950, title = {Audiovisual synchrony detection for speech and music signals}, year = {2009}, month = {7}, volume = {10}, number = {689}, pages = {177-178}, abstract = {Introduction: Audiovisual integration crucially depends on the relative timing of the auditory and visual signals. Although multisensory signals do not have to be precisely physically synchronous in order to be perceived as single temporal events, they have to co-occur within a certain temporal window of integration. To investigate how the human brain is fine tuned to the natural temporal statistics of audiovisual signals, we characterized the temporal integration window for natural speech, sinewave replicas of natural speech (SWS) and music in a simultaneity judgment task. Methods: The experimental paradigm manipulated: 1) stimulus class: speech vs. SWS vs. music, and 2) stimulus length: short (i.e. natural syllables, SWS syllables and tones) vs. long (i.e. natural sentences, SWS sentences and melodies). Audiovisual asynchronies ranged from -360ms (auditory leading) to 360 ms (visual leading) in 60ms increments. Eight participants performed the experiment on 2 separate days. The order of conditions was counterbalanced within and between subjects. The proportion of synchronous responses was computed for each participant. To refrain from making any distributional assumptions, the psychometric curves of each participant were characterized by four indices: (i) peak performance, (ii) peak location, (iii) width and (iv) asymmetry [1]. The four indices were analyzed using repeated measures of ANOVAs with stimulus class and stimulus length as within-subjects factors. Results: The ANOVA for peak performance did not show any significant main effects of stimulus class and length [F(2,14)<1, n.s.; F(1,7)=1.6, p=.24]. The ANOVA for peak location revealed a significant interaction between stimulus class and length [F(2,14)=3.8, p<.05]. Post-hoc paired t-tests revealed that the peak locations were significantly shifted towards auditory leading for melodies compared to tones [t(7)=2.4, p<.05], and for melodies compared to SWS sentences [t(7)=-2.3, p=.053]. The ANOVA for width revealed significant main effects of stimulus class and length [F(2,14)=9.3, p<.005; F(1,7)=11.0, p<.05] in the absence of an interaction [F(2,14)<1, n.s.]. Post-hoc paired t-tests revealed that the widths were wider for SWS speech than natural speech [t(7)=7.0, p<.005] and music [t(7)=2.4, p=.05]. Furthermore, the widths were narrower for long stimuli (i.e. sentences and melodies) than short stimuli (i.e. syllables and tones) [t(7)=-3.3, p<.05]. With respect to the asymmetry, there was a significant main effect of stimulus length [F(1,7)=7.1, p<.05] but not stimulus class [F(2,14)=1.1, p=.35], thus indicating that the psychometric curves were more asymmetric for long stimuli (i.e. sentences and melodies) than short stimuli (i.e. syllables and tones). Conclusion: Our results demonstrated that the psychometric curves were narrower and more asymmetric for long stimuli (i.e. sentences and melodies) than short stimuli (i.e. syllables and tones). Thus, participants may rely on information during the entire sentence for synchrony judgments. In addition, our results demonstrated that the psychometric curves were wider but less asymmetric for SWS speech relative to natural speech and music. Collectively, our results support the hypothesis that audiovisual speech perception is fine-tuned to the natural mapping between facial movement and spectrotemporal structure of natural speech.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs/index.php/meetings/2009/paper/view/689}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {New York, NY, USA}, event_name = {10th International Multisensory Research Forum (IMRF 2009)}, language = {en}, author = {Lee, H and Noppeney, U} } @Poster { 5949, title = {Auditory influences on the temporal dynamics of binocular rivalry}, year = {2009}, month = {7}, volume = {10}, number = {590}, pages = {102-103}, abstract = {Introduction When the two eyes are presented with dissimilar images, human observers report alternating percepts {\^a}{\euro}“ a phenomenon coined binocular rivalry. These perceptual fluctuations reflect competition between the two visual inputs both at lower, monocular and at binocular, higher-level processing stages. Even though perceptual transitions occur stochastically over time, their temporal dynamics can be modulated by changes in stimulus strength, context and attention. While increases in stimulus strength (such as contrast) primarily abbreviate suppression phases of a percept, attentional and contextual factors predominantly lengthen its dominance periods. Goals This project investigates the influence of concurrent auditory stimulation on the temporal dynamics of binocular rivalry. In two psychophysics studies, we investigated whether sounds that provide directionally congruent, incongruent or no motion information modulate the dominance periods of rivaling visual motion percepts. Methods In the first psychophysics study, observers dichoptically viewed random-dot kinematograms (RDK) at 0\% motion coherence in one eye and 50\% in the other in a stereoscope, while being concurrently presented with directionally congruent auditory motion, noise and no sound. In the second psychophysics study, they viewed two RDKs of opposite motion directions at 100\% coherence, with the auditory motion stimulus being directionally congruent with one of the two rivaling motion percepts. In both experiments, congruent auditory motion was temporally synchronized with visual motion to facilitate audio-visual integration into a coherent percept. Initial results Both experiments consistently revealed a statistically significant influence of sound on perceptual dominance times. In the first experiment, directionally congruent auditory motion but not noise increased the duration of the dominance phases of the RDK at 50\% motion coherence. In the second experiment, auditory motion lengthened the dominance periods of the directionally congruent 100\% RDK and abbreviated those of the directionally incongruent 100\% RDK. Initial conclusions The results demonstrate that auditory stimuli influence the temporal dynamics of binocular rivalry. Auditory motion lengthened the dominance periods of a visual motion percept when it was directionally congruent, but shortened them when it was directionally incongruent. Thus, the (in)congruency of auditory motion primarily influences the duration of the dominance periods similar to purely visual contextual effects, even though a small effect was also observed on the suppression periods. In conclusion, the human brain draws on information from multiple senses to arbitrate between multiple rivaling perceptual interpretations.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs/index.php/meetings/2009/paper/view/590}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {New York, NY, USA}, event_name = {10th International Multisensory Research Forum (IMRF 2009)}, language = {en}, author = {Conrad, V and Bartels, A and Kleiner, M and Noppeney, U} } @Poster { 5934, title = {The contributions of transient and sustained responses to audio visual integration of dynamic information}, year = {2009}, month = {7}, volume = {10}, number = {754}, pages = {251-252}, abstract = {Transient and sustained responses have been shown to play distinct functional roles in auditory processing: Transient responses may subserve rapid stimulus detection, sustained responses contribute to a more detailed sound characterization. While numerous fMRI studies have reported audiovisual interactions at multiple levels of the cortical hierarchy, they were not able to dissociate transient and sustained responses. This fMRI study optimized the design to disentangle the contributions of sustained, onset and offset responses to superadditive and subadditive interactions and localize the effects within the visual and auditory processing hierarchies. Seventeen subjects participated in this fMRI study (Siemens TimTrio 3T scanner, GE-EPI, TE = 40 ms, 38 axial slices, TR = 3.08 s). While engaged in a target detection task, they were presented with 1 s, 10 s, 20 s, 30 s blocks of (i) video clips of an expanding radial star-field, (ii) auditory pink noise or (iii) both. The velocity of the star-field and the sound amplitude were jointly modulated according to 0.1 Hz sine-wave function. The regressors of the general linear model were formed by convolving (i) delta functions encoding the onset and offset of each block and (ii) box car functions adjusted for block length with the hemodynamic response functions. Blocks of 1 s duration were modeled only as onsets. In addition, the model included targets and parametric modulators encoding the amplitude / velocity modulation. To allow for a random-effects analysis (SPM5), contrast images for each subject were entered into second level one sample t-tests. We tested for superadditive and subadditive interactions separately for onset, offset and sustained block responses. Results are reported at p<0.05 whole brain corrected. Significant audiovisual interactions were observed only for the transients: For the onsets, the interactions were superadditive in the fusiform gyrus (FFG), anterior calcarine sulcus (aCaS) and the cuneus (Cun) and subadditive in the posterior superior temporal gyrus/sulcus (pSTS/STG) and the precuneus (PrCun). For the offsets, the interactions were subadditive in the pSTS/STG region and the anterior intraparietal sulcus (aIPS). The regional response profiles were further characterized by their general responsiveness to visual, auditory and audiovisual onsets, offsets and sustained stimulation. This dissociated three activation profiles: (i) In FFG, only the onsets elicited a strong positive response with moderate responses to offsets and sustained stimulation. Further, the onset responses were positive for visual and audiovisual stimuli and negative for auditory stimuli. (ii) In aCaS, only the offsets elicited a positive response for all sensory modalities. (iii) In the remaining regions, both onsets and offsets elicited a positive response for all sensory modalities. In conclusion, audiovisual interactions are observed primarily for transient rather than sustained stimulation. Furthermore, these AV interactions are located in regions that respond primarily to transients. In contrast, no significant interactions were observed in regions that exhibited sustained responses to extended blocks of audiovisual stimulation.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/PosterIMRF2009_FINAL_[0].pdf}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs/index.php/meetings/2009/paper/view/754}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {New York, NY, USA}, event_name = {10th International Multisensory Research Forum (IMRF 2009)}, language = {en}, author = {Werner, S and Noppeney, U} } @Poster { RotshteinDBRNH2009, title = {The neural correlates of picture naming: a VBM study of CT images}, journal = {NeuroImage}, year = {2009}, month = {7}, volume = {47}, number = {Supplement 1}, pages = {S174}, abstract = {Introduction Neuropsychological researchers have long been interested in relating the location of a patient's lesion to her cognitive impairment (e.g. Broca, 1861). Previous studies have primarily relied on small patient numbers, manual procedures for symptom-lesion mapping and binary data (Davidoff and De, 1994; DeLeon et al., 2007; Luzzatti et al., 2006; Tranel et al., 1997). Furthermore, the above studies did not control for potential confounds due to covarying factors, such as levels of attention; all of these factors may have contributed to the inconsistencies in the current literature. Here, we combined robust, automated lesion analysis procedures and standard CT imaging to investigate the neural networks involved in object naming in a clinically-relevant setting. Extensive prior research suggests that structures within the left ventral visual stream, anterior temporal and frontal cortex play a key role in mediating object naming (Humphreys et al., 1999; Martin, 2007; Price et al., 1996). Methods We tested 80 acute stroke patients using the object naming task from the Birmingham University Cognitive Screening (BUCS(Humphrey et al., 2007)). All patients underwent a CT scan as part of their clinical evaluation. Data were analysed using SPM5 \& SPM8b. The CT images were normalized to a CT template, the skull was removed. Subsequently the images were segmented and normalized again; finally the segmented grey matter (GM) images were smoothed using a 12mm Gaussian kernel (Fig. 1). The GM images were entered into a regression analysis that modeled subjects’ performance on the picture naming task; age, gender, mini-mental state and depression were included as covariates of no interest. (1) Using mass-univariate voxel-based morphometry (VBM), we tested at every single voxel for reduction in GM volume that was positively predicted by subjects’ picture naming performancen. (2) Combining Multivariate Bayesian Decoding (MVB(Friston et al., 2008) with smooth priors) and Bayesian model comparison, we compared the ability of GM voxels within (i) the entire brain, (ii) the left occipito-temporal, and (iii) left frontal cortices to predict subject's naming performance. Results On average patients named 9.1/14 ± 3.64std pictures correctly. When compared to age matched healthy controls, 42 patients were classified as impaired (<10, Fig. 2). VBM revealed that impaired picture naming predicted reduced GM in the left ventral visual stream, the anterior temporal pole and (bi-laterally) in the middle frontal gyrus (P <0.001, Fig 3). MVB and Bayesian model comparison showed that the left frontal gyrus did not predict behaviour better than the null model (log evidence (LG) <3). In contrast, the left occipito-temporal cortices were significantly better predictors than the null model (LG > 5, Fig 4). Conclusions In line with previous research we showed that lesions to the left ventral visual stream and frontal cortices impaired the ability to name objects. Using MVB we found that lesions within the left occipito-temporal cortices were better predictors of behaviour than lesions to the left frontal. We conclude that CT data of large number of patients can provide useful information for function-lesion mapping, here used to reveal the neural circuits involved in picture naming.}, department = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811909718890}, event_place = {San Francisco, CA, USA}, event_name = {15th Annual Meeting of the Organisation for Human Brain Mapping (HBM 2009)}, DOI = {10.1016/S1053-8119(09)71889-0}, author = {Rotshtein, P and Douis, H and Bickerton, W and Riddoch, JM and Noppeney, U and Humphreys, GW} } @Poster { 6297, title = {Visual object categorization with conflicting auditory information}, journal = {NeuroImage}, year = {2009}, month = {7}, volume = {47}, number = {Supplement 1}, pages = {S43}, abstract = {Introduction Spatio-temporally coincident sensory signals usually emanate from the same event or object. Ideally, the human brain should integrate sensory information derived from a common source, while avoiding mergence of information from different sources. Indeed, it has long been recognized that multisensory integration breaks down, when sensory estimates are brought into large conflict. Nevertheless, conflicting task-irrelevant sensory information has been shown to interfere with decisions on task-relevant sensory input in a selective crossmodal attention paradigm. Combining psychophysics and fMRI, this study investigates how the human brain forms decisions about multisensory object categories when the senses disagree. Methods 18 subjects participated in this fMRI study (Siemens Trio 3T scanner, GE-EPI, TE=40 ms, 38 axial slices, TR=3.08 s). The 2\(\times\)2 factorial design manipulated (i) Visual category: Animal vs. Landmark, and (ii) Auditory category: Animal vocalization vs. Sound associated with Landmark. In a visual selective attention paradigm, subjects categorized degraded pictures while ignoring their accompanying intact sounds that could be either semantically congruent or incongruent. These particular stimulus categories were used as they have been associated with selective activations in the fusiform face (FFA) vs. parahippocampal place (PPA) area. To localize FFA and PPA, subjects were also presented pictures of faces and landmarks in target detection task. The activation trials were interleaved with 6 s fixation blocks. To allow for a random-effects analysis (SPM5), contrast images for each subject were entered into a second level ANOVA. In additional regression analyses, subjects’ category-selective responses (localizer) were regressed on their reaction time (RT) increases for incongruent relative to congruent trials (separately for landmarks and animals). We tested for (1) Incongruent vs. Congruent, (2) Landmark vs Animal (3) Incongruency \(\times\) Category interaction and (4) Landmark vs Animal predicted by RT incongruency effects. Within the object processing system, results are reported at p<0.05 whole brain cluster level corrected with p<0.001 voxel threshold. Results Behaviour: 2-way ANOVA of RT revealed main effect of incongruency, in the absence of a main effect of category or an interaction. No significant effects were revealed for performance accuracy. fMRI: 1) Incongruency effects: left inferior precentral sulcus, bilateral insula (only at uncorrected threshold) 2) Landmark-selective: bilateral PPA; Animal-selective: bilateral FFA 3) Incongruent > Congruent for Landmark > Animal: bilateral auditory cortex (due to stimulus confounds) 4) Landmark > Animal predicted by RT incongruency effects for landmarks: left fusiform Conclusions At the behavioural level, conflicting task-irrelevant auditory information interfered with subjects' categorization as indicated by longer reaction times for incongruent relative to congruent stimuli. At the neural level, this behavioural interference was mediated by increased activation in bilateral insula and left inferior precentral sulcus. No incongruency effects were observed commonly for all subjects in category-selective areas. However, in the left fusiform (FFA), activation for Landmark > Animal was increased for subjects that showed strong behavioural interference during categorization of visual landmarks accompanied by animal sounds. Thus, less modular and more distributed object representations in the occipitotemporal cortex may render subjects more susceptible to interfering object information from the auditory modality.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/Adam_HBM2009_5uta_[0].ppt}, department = {Research Group Noppeney}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {San Francisco, CA, USA}, event_name = {15th Annual Meeting of the Organisation for Human Brain Mapping (HBM 2009)}, language = {en}, DOI = {10.1016/S1053-8119(09)70022-9}, author = {Adam, R and Noppeney, U} } @Poster { 5948, title = {Visual-Auditory Synchrony Boosts BOLD Response in Posterior Temporal and Occipital Cortices}, year = {2009}, month = {7}, volume = {10}, number = {571}, pages = {90-91}, abstract = {Introduction: Synchrony is a powerful cue for driving multisensory integration of dynamic stimuli and multisensory integration increases perceptual reliability. Thus ambiguous, dynamic, visual targets should be more accurately perceived when accompanied by concurrent auditory stimuli. For example, when a creature moving through the undergrowth is obscured by both intervening foliage \& conditions of poor illumination, it may be more readily identified when the sound of each footfall is audible - heralding motion of figure against background and foreground. This fMRI study was designed to identify the neural correlates of synchrony-induced multisensory integration during shape and motion discrimination. Methods: 16 subjects participated in this fMRI study (Siemens Allegra 3 T scanner, GE-EPI, TE=30, 38 axial slices, TR=3.08s). The 2x2 factorial design manipulated audiovisual synchrony (synchronous vs. asynchronous) and task (motion vs. shape discrimination). In a visual selective attention paradigm, subjects discriminated the shape or rotational motion of a dot array obscured by a rapidly-moving snow field. Each 2.5s trial consisted of 5 visual events that occurred at unpredictable intervals to eliminate anticipatory effects associated with regular stimulus timings. In 50\% of the trials, each visual event was accompanied by a simultaneous 50ms {\^a}{\euro}œclick{\^a}{\euro}? sound (synchronous condition), whilst in the other 50\% the auditory and visual events were offset by at least 120ms (asynchronous condition). Visual events were discrete rotations of one of 32 different symmetrical, concentrically distributed, dot arrays. Synchronous and asynchronous trials were presented in randomised order and task order was counterbalanced within and across subjects. To allow for a random-effects analysis (SPM5), contrast images for each subject were entered into second level one-sample t-tests. We tested for the main effects of synchrony and task and their interactions. Effects are reported at p<0.05 whole brain corrected. Behavioural Results: Subject performance was significantly more accurate under conditions of visual-auditory synchrony than asynchrony, whilst there was no significant difference in performance between the shape and motion tasks. The interactions indicate a significant performance improvement during synchronous trials for the motion task, but not the shape task. fMRI Results: In bilateral posterior occipital cortices audiovisual synchrony amplified the BOLD response, irrespective of task. In contrast, in lateral occipital cortex (LOC) and posterior superior temporal (pSTG) / supramarginal (SMG) gyri, synchrony-induced activation increases were task dependent: in LOC synchrony effects were selective for shape discrimination, whilst in pSTG/SMG they were selective for motion discrimination. Discussion/conclusion: Our results suggest that, under noisy visual conditions, visual perception can be enhanced by concurrent acoustic stimulation in the following way: Audiovisual coincidence may enhance saliency of visual stimuli by amplification of visual responses in occipital cortex and thus lead to improved figure-ground segmentation. These are then further amplified in higher order LOC and pSTG/SMG in a task-selective fashion.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs/index.php/meetings/2009/paper/view/571}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {New York, NY, USA}, event_name = {10th International Multisensory Research Forum (IMRF 2009)}, language = {en}, author = {Lewis, RK and Noppeney, U} } @Poster { CappelettiLFP2009, title = {The Role of Right and Left Parietal Lobes in the Conceptual Processing of Numbers}, journal = {Journal of Cognitive Neuroscience}, year = {2009}, month = {3}, volume = {21}, number = {Supplement}, pages = {35}, abstract = {Neuropsychological and functional imaging studies have associated the conceptual processing of numbers with bilateral parietal regions (including the intraparietal sulcus, IPS). However, the processes driving these effects remain unclear because both left and right posterior parietal regions are activated by many other conceptual, perceptual, attention and response-selection processes. To dissociate parietal activation that is number-selective from parietal activation related to other stimulus or response-selection processes, we used fMRI to compare numbers and object names during exactly the same conceptual and perceptual tasks while factoring out activations correlating with response times. We found that right parietal activation was higher for conceptual decisions on numbers relative to the same tasks on object names, even when response-time effects were fully factored out. In contrast, left parietal activation for numbers was equally involved in conceptual processing of object names. We suggest that left parietal activation for numbers reflects a range of processes, including the retrieval of learnt facts that are also involved in conceptual decisions on object names. In contrast, number-selectivity in the right parietal cortex reflects processes that are more involved in conceptual decisions on numbers than object names. Our results generate a new set of hypotheses that have implications for the design of future behavioural and functional imaging studies of patients.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://cogneurosociety.org/annual-meeting/previous-meetings/CNS2009_Program.pdf/view}, event_place = {San Francisco, CA, USA}, event_name = {16th Annual Meeting of the Cognitive Neuroscience Society (CNS 2009)}, author = {Cappeletti, M and Lee, HL and Freeman, ED and Price, CJ} } @Conference { 6298, title = {Auditory influence on visual object categorization: an fMRI study}, journal = {Journal of Molecular Neuroscience}, year = {2009}, month = {11}, volume = {39}, number = {The Israel Society for Neuroscience 18th Annual Meeting Supplement}, pages = {4-5}, abstract = {In our daily life, we are often confronted with objects that give rise to signals in multiple sensory modalities. The brain’s challenge is to integrate information from multiple senses into a unified percept. Combining psychophysics and fMRI, the present study investigates how task-irrelevant object source sounds affect category-selective activations for visual faces in the fusiform and landmarks in the parahippocampal gyri. In a visual selective attention paradigm, subjects categorized degraded object pictures as landmarks or faces while ignoring intact object source sounds that were either semantically congruent or incongruent. The 2X2 factorial design manipulated (i) Visual category: Animal vs. Landmark, and (ii) Auditory category: Animal vocalization vs. Sound associated with Landmark. Behaviorally, incongruent trials were associated with longer response times. This effect emerged due to an interference of incongruent sounds associated with landmarks on the categorization of visual faces. At the neural level, only landmark-selective activation in the parahippocampal gyrus, but not face selective activations in the fusiform gyrus were modulated by the congruency of an irrelevant object sound. More specifically, activations in the anterior parahippocampal gyrus showed additive effects of both visual and auditory category information. Effective connectivity analysis indicated that parahippocampal responses are amplified by incongruent auditory inputs via enhanced coupling between auditory and occipito-temporal cortices. Collectively, these results suggest that a region in the parahippocampal gyrus integrates information about object categories from multiple senses. In contrast, face-selective responses in fusiform gyrus are more robust and less influenced by task-irrelevant sounds, even in the context of a behavioural audiovisual interference effect.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://springerlink.com/content/942767xp230h16r5/fulltext.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, author = {Adam, R and Noppeney, U} } @Conference { 5951, title = {Inverse effectiveness in BOLD-response and its behavioural relevance in object categorization}, year = {2009}, month = {7}, volume = {10}, pages = {395}, abstract = {Inverse effectiveness has been invoked as a principle to describe synergistic effects of multisensory integration in neuronal and behavioural responses as a function of stimulus properties (e.g. intensity) or efficacy. We characterized {\^a}{\euro}˜inverse effectiveness{\^a}{\euro}™ and its behavioural relevance at the macroscopic level, as provided by the fMRI BOLD-response, based on (1) stimulus-induced and (2) intrinsic response variability across voxels or subjects during object categorization. Subjects categorized audiovisual object stimuli with the relative informativeness (i.e. degradation) of the auditory and visual inputs being manipulated factorially. Controlling for low-level integration processes, higher-level audiovisual integration was observed selectively in the superior temporal sulci (STS) bilaterally. (1) Consistent with the law of inverse effectiveness, auditory and visual informativeness determined the operational modes of audiovisual integration in STS similarly to the influence of physical stimulus intensity in the superior colliculus: while multisensory interactions were primarily subadditive and even suppressive for intact stimuli, additive effects were observed for degraded, near threshold stimuli. (2) Exploiting intrinsic variability across voxels and/or subjects, we demonstrate that superadditivity for audiovisual stimuli increases with decreasing unimodal responses. This inverse relationship could be explained by inherent statistical dependencies between superadditive and unimodal responses. Nevertheless, the superadditive responses in STS (and only in this region) were related to subjects{\^a}{\euro}™ audiovisual behavioral benefit: only subjects that benefited from multisensory integration exhibited superadditive interactions, while those that did not benefit showed suppressive interactions. In conclusion, the (super)additive and subadditive integration modes in STS are functionally relevant and related to behavioral indices of multisensory integration with superadditive interactions mediating successful audiovisual object categorization. We argue that inverse effectiveness trends in neuronal and behavioural responses may be intimately related and mutually predictive.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://imrf.mcmaster.ca/IMRF/ocs/index.php/meetings/2009/paper/view/897}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {New York, NY, USA}, event_name = {10th International Multisensory Research Forum (IMRF 2009)}, language = {en}, author = {Noppeney, U and Werner, S} } @Article { 5138, title = {The integration of higher order form and motion by the human brain}, journal = {NeuroImage}, year = {2008}, month = {10}, volume = {42}, number = {4}, pages = {1529-1536}, abstract = {Our experience with a dynamic environment has tuned our visual system to use form and motion as complementary sources of information for object recognition. To identify the neural systems involved in integrating form and motion information during dynamic object processing, we used an fMRI adaptation paradigm which factorially manipulated form and motion repetition. Observers were sequentially presented with pairs of rotating novel objects in which the form or rotation direction in depth could be repeated. They were required to discriminate either dimension of the second target object, while the first object served as a form or motion prime. At the behavioural level, observers were faster to recognize the target or discriminate its direction when primed by the same form. Importantly, this form priming effect was enhanced when prime and target objects rotated in the same direction. At the neural level, the two priming effects (i.e., the main effect of form repetition and the interaction between form and motion repetition) were associated with reduced activations in distinct brain regions. Bilateral lateral occipital regions exhibited reduced activation when form was repeated irrespective of rotation direction. In contrast, bilateral anterior fusiform and posterior middle temporal regions (overlapping with hMT+/V5) regions showed an adaptation effect that depended on both form and motion direction. Thus, the current results reveal a visual processing hierarchy with lateral occipito-temporal cortex representing an object’s 3D structure, and anterior fusiform and posterior middle temporal regions being involved in spatio-temporal integration of form and motion during dynamic object processing.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science?_ob=MImg\&_imagekey=B6WNP-4SH0Y0C-2-M\&_cdi=6968\&_user=29041\&_orig=search\&_coverDate=10\%2F01\%2F2008\&_sk=999579995\&view=c\&wchp=dGLbVtb-zSkWb\&md5=2afffb798d11cb801968184b420b9242\&ie=/sdarticle.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1016/j.neuroimage.2008.04.265}, author = {Sarkheil, P and Vuong, QC and B{\"u}lthoff, HH and Noppeney, U} } @Article { 5379, title = {Imaging seizure activity: A combined EEG/EMG-fMRI study in reading epilepsy}, journal = {Epilepsia}, year = {2008}, month = {8}, volume = {50}, number = {2}, pages = {256-264}, abstract = {Purpose: To characterize the spatial relationship between activations related to language-induced seizure activity, language processing, and motor control in patients with reading epilepsy. Methods: We recorded and simultaneously monitored several physiological parameters [voice-recording, electromyography (EMG), electrocardiography (ECG), electroencephalography (EEG)] during blood oxygen level-dependent (BOLD) functional magnetic resonance imaging (fMRI) in nine patients with reading epilepsy. Individually tailored language paradigms were used to induce and record habitual seizures inside the MRI scanner. Voxel-based morphometry (VBM) was used for structural brain analysis. Reading-induced seizures occurred in six out of nine patients. Results: One patient experienced abundant orofacial reflex myocloni during silent reading in association with bilateral frontal or generalized epileptiform discharges. In a further five patients, symptoms were only elicited while reading aloud with self-indicated events. Consistent activation patterns in response to reading-induced myoclonic seizures were observed within left motor and premotor areas in five of these six patients, in the left striatum (n = 4), in mesiotemporal/limbic areas (n = 4), in Brodmann area 47 (n = 3), and thalamus (n = 2). These BOLD activations were overlapping or adjacent to areas physiologically activated during language and facial motor tasks. No subtle structural abnormalities common to all patients were identified using VBM, but one patient had a left temporal ischemic lesion. Discussion: Based on the findings, we hypothesize that reflex seizures occur in reading epilepsy when a critical mass of neurons are activated through a provoking stimulus within corticoreticular and corticocortical circuitry subserving normal functions.}, department = {Research Group Noppeney}, web_url = {http://www3.interscience.wiley.com/cgi-bin/fulltext/121383967/PDFSTART}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1111/j.1528-1167.2008.01737.x}, author = {Salek-Haddadi, A and Mayer, T and Hamandi, K and Symms, M and Josephs, O and Fluegel, D and Woermann, F and Richardson, MP and Noppeney, U and Wolf, P and Koepp, MJ} } @Article { 4610, title = {Selective activation around the left occipito-temporal sulcus for words relative to pictures: individual variability or false positives?}, journal = {Human Brain Mapping}, year = {2008}, month = {8}, volume = {29}, number = {8}, pages = {986-1000}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www3.interscience.wiley.com/cgi-bin/fulltext/115806086/PDFSTART}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1002/hbm.20443}, author = {Wright, ND and Mechelli, A and Noppeney, U and Veltman, DJ and Rombouts, SARB and Glensman, J and Haynes, J-D and Price, CJ} } @Article { 5174, title = {The neural systems of tool and action semantics: A perspective from functional imaging}, journal = {Journal of Physiology - Paris}, year = {2008}, month = {5}, volume = {102}, number = {1-3}, pages = {40-49}, abstract = {This review discusses the contributions of functional imaging (fMRI/PET) to our understanding of how action and tool concepts are represented and processed in the human brain. Category-selective deficits in neuropsychological patients have suggested a fine-grained functional specialization within the neural systems of semantics. However, the underlying principles of semantic organization remain controversial. The feature-based account of semantic memory (or ‘sensory-motor theory’) predicates category-selective effects (e.g. tool vs. animals) on anatomical segregation for different semantic features (e.g. action vs. visual). Within this framework, we will review functional imaging evidence that semantic processing of tools and actions may rely on activations within the visuo-motor system.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science?_ob=MImg\&_imagekey=B6VMC-4S7JFVY-1-7\&_cdi=6147\&_user=29041\&_orig=search\&_coverDate=05\%2F31\%2F2008\&_sk=998979998\&view=c\&wchp=dGLbVzW-zSkzS\&md5=840b72da18aaa4abd82963b14c7ee9ae\&ie=/sdarticle.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1016/j.jphysparis.2008.03.009}, author = {Noppeney, U} } @Article { 4600, title = {The effect of prior visual information on recognition of speech and sounds}, journal = {Cerebral Cortex}, year = {2008}, month = {3}, volume = {18}, number = {3}, pages = {598-609}, abstract = {To identify and categorize complex stimuli such as familiar objects or speech, the human brain integrates information that is abstracted at multiple levels from its sensory inputs. Using cross-modal priming for spoken words and sounds, this functional magnetic resonance imaging study identified 3 distinct classes of visuoauditory incongruency effects: visuoauditory incongruency effects were selective for 1) spoken words in the left superior temporal sulcus (STS), 2) environmental sounds in the left angular gyrus (AG), and 3) both words and sounds in the lateral and medial prefrontal cortices (IFS/mPFC). From a cognitive perspective, these incongruency effects suggest that prior visual information influences the neural processes underlying speech and sound recognition at multiple levels, with the STS being involved in phonological, AG in semantic, and mPFC/IFS in higher conceptual processing. In terms of neural mechanisms, effective connectivity analyses (dynamic causal modeling) suggest that these incongruenc y effects may emerge via greater bottom-up effects from early auditory regions to intermediate multisensory integration areas (i.e., STS and AG). This is consistent with a predictive coding perspective on hierarchical Bayesian inference in the cortex where the domain of the prediction error (phonological vs. semantic) determines its regional expression (middle temporal gyrus/STS vs. AG/intraparietal sulcus).}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/bhm091v1_noppeney_[0].pdf}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://cercor.oxfordjournals.org/cgi/reprint/bhm091v1?maxtoshow=\&HITS=10\&hits=10\&RESULTFORMAT=1\&author1=noppeney\&andorexacttitle=and\&andorexacttitleabs=and\&andorexactfulltext=and\&searchid=1\&a}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/cercor/bhm091}, author = {Noppeney, U and Josephs, O and Hocking, J and Price, CJ and Friston, KJ} } @Poster { 5263, title = {Audio-visual object integration in human STS: Determinants of stimulus efficacy and inverse effectiveness}, year = {2008}, month = {7}, volume = {9}, pages = {275}, abstract = {Combining fMRI and psychophysics, we investigated the neural mechanisms underlying the integration of higher-order audio-visual object features. In a target detection and a semantic categorization task, we presented subjects with pictures and sounds of tools or musical instruments while factorially manipulating the relative informativeness (degradation) of auditory and visual stimuli. Controlling for integration effects of low-level stimulus features, our experiment reveals integration of higher-order audio-visual object information selectively in anterior and posterior STS regions. Across subjects, audio-visual BOLD-interactions within these regions were strongly subadditive for intact stimuli and turned into additive effects for degraded stimuli. Across voxels, the probability to observe subadditivity increased with the strength of the unimodal BOLD-responses for both degraded and intact stimuli. Importantly, subjects’ multi-sensory behavioural benefit significantly predicted the mode of integration in STS : Subjects with greater benefits exhibited stronger superadditivity. In conclusion and according to the inverse effectiveness principle that is determined by stimulus efficacy, we demonstrate that the mode of multi-sensory integration in STS depends on stimulus informativeness, the voxel-specific responsiveness to unimodal stimulus components and the subject-specific multi-sensory behavioural benefit in object perception. The relationship between BOLD-responses and behavioural indices show the functional relevance of super- and subadditive modes of multi-sensory integration.}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/IMRF2008_Swerner_[0].pdf}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/2008/pdf/FullProgramIMRF08.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Hamburg, Germany}, event_name = {9th International Multisensory Research Forum (IMRF 2008)}, language = {en}, author = {Werner, S and Noppeney, U} } @Poster { 5479, title = {Natural, metaphoric and linguistic auditory-visual interactions}, year = {2008}, month = {7}, volume = {9}, number = {206}, pages = {132}, abstract = {To form a coherent percept of our dynamic environment, the brain merges motion information from the auditory and visual senses. Yet, not only auditory motion, but also {\^a}{\euro}˜metaphoric{\^a}{\euro}™ pitch has been shown to influence visual motion discrimination. Here, we systematically investigate the neural systems that mediate auditory influences on visual motion discrimination in natural, metaphoric and linguistic contexts. In a visual selective attention paradigm, subjects discriminated the direction of visual motion at several levels of ambiguity, while ignoring a simultaneous auditory stimulus that was 1) {\^a}{\euro}˜natural{\^a}{\euro}™ MOTION: left vs. right moving white noise, 2) {\^a}{\euro}˜metaphoric{\^a}{\euro}™ PITCH: rising vs. falling pitch or 3) {\^a}{\euro}˜linguistic{\^a}{\euro}™ SPEECH: spoken German words denoting directions e.g. {\^a}{\euro}˜links{\^a}{\euro}™ vs. {\^a}{\euro}˜rechts{\^a}{\euro}™. Behaviourally, all three classes of auditory stimuli induced a comparable directional bias. At the neural level, the interaction between visual ambiguity and audition revealed an auditory influence on visual motion processing for MOTION in left hMT/V5 and for SPEECH in right intraparietal sulcus. Direct comparisons across contexts confirmed this functional dissociation: The interaction effect gradually decreased in left hMT+/V5 for MOTION>PITCH>SPEECH and in right IPS for SPEECH>PITCH>MOTION. In conclusion, while natural audio-visual integration of motion signals emerges in motion processing areas, linguistic interactions are revealed primarily in higher level fronto-parietal regions.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/2008/pdf/FullProgramIMRF08.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Hamburg, Germany}, event_name = {9th International Multisensory Research Forum (IMRF 2008)}, language = {en}, author = {Sadaghiani, S and Maier, J and Noppeney, U} } @Poster { 5267, title = {Physical and perceptual factors that determine the mode of audio-visual integration in distinct areas of the speech processing system}, year = {2008}, month = {7}, volume = {9}, number = {208}, pages = {133}, abstract = {Speech and non-speech stimuli differ in their (i) physical (spectro-temporal structure) and (ii) perceptual (phonetic/linguistic representation) aspects. To dissociate these two levels in audio-visual integration, this fMRI study employed original spoken sentences and their sinewave analogues that were either trained and perceived as speech (group 1) or non-speech (group 2). In both groups, all stimuli were presented in visual, auditory or audiovisual modalities. AV-integration areas were identified by superadditive and subadditive interactions in a random effects analysis. While no superadditive interactions were observed, subadditive effects were found in right superior temporal sulci for both speech and sinewave stimuli. The left ventral premotor cortex showed increased subadditive interactions for speech relative to their sinewave analogues irrespective of whether they were perceived as speech or non-speech. More specifically, only familiar auditory speech signal suppressed premotor activation that was elicited by passive lipreading in the visual conditions, suggesting that acoustic rather than perceptual/linguistic features determine AV-integration in the mirror neuron system. In contrast, AV-integration modes differed between sinewave analogues perceived as speech and non-speech in bilateral anterior STS areas that have previously been implicated in speech comprehension. In conclusion, physical and perceptual factors determine the mode of AV-integration in distinct speech processing areas.}, department = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/2008/pdf/FullProgramIMRF08.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Hamburg, Germany}, event_name = {9th International Multisensory Research Forum (IMRF 2008)}, language = {en}, author = {Lee, HL and Tuennerhoff, J and Werner, S and Pammi, C and Noppeney, U} } @Poster { 4906, title = {Differences in the neural basis of Chinese and English reading}, journal = {UCL-China Research Festival}, year = {2008}, month = {6}, volume = {2008}, pages = {13}, department = {Research Group Noppeney}, web_url = {http://www.ucl.ac.uk/global/china/CRF-08/images/CRF-08_abstracts.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, author = {Green, DW and Hu, W and Lee, HL and Zhang, Q and Liu, T and Libo, G and Seghier, ML and Price, CJ} } @Poster { 5265, title = {The prefrontal cortex accumulates object evidence through differential connectivity to the visual and auditory cortices}, journal = {NeuroImage}, year = {2008}, month = {6}, volume = {41}, number = {Supplement 1}, pages = {S150}, department = {Research Group Noppeney}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Melbourne, Australia}, event_name = {14th Annual Meeting of the Organization for Human Brain Mapping (HBM 2008)}, language = {en}, DOI = {10.1016/j.neuroimage.2008.04.008}, author = {Noppeney, U and Ostwald, D and Kleiner, M and Werner, S} } @Conference { 5264, title = {The prefrontal cortex accumulates object evidence through differential connectivity to the visual and auditory cortices}, year = {2008}, month = {7}, volume = {9}, number = {189}, pages = {118}, abstract = {To form categorical decisions about objects in our environment, the human brain accumulates noisy sensory information over time till a decisional threshold is reached. Combining fMRI and Dynamic Causal Modelling (DCM), we investigated how the brain accumulates evidence from the auditory and visual senses through distinct interactions amongst brain regions. In a visual selective attention paradigm, subjects categorized visual action movies while ignoring their accompanying soundtracks that were semantically congruent or incongruent. Both, auditory and visual information could be intact or degraded. Reaction times as a marker for the time to decisional threshold accorded with random walk models of decision making. At the neural level, incongruent auditory sounds induced amplification of the task-relevant visual information in the occipito-temporal cortex. Importantly, only the left inferior frontal sulcus (IFS) showed an activation pattern of an accumulator region i.e. (i) positive reactiontime and (ii) incongruency effects that were increased for unreliable (=degraded) visual and interfering reliable (=intact) auditory information, which -based on our DCM analysis- were mediated by increased forward connectivity from visual regions. Thus, to form interpretations and decisions that guide behavioural responses, the IFS may accumulate multi-sensory evidence over time through dynamic weighting of its connectivity to auditory and visual regions.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://imrf.mcmaster.ca/IMRF/2008/pdf/FullProgramIMRF08.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, institution = {Max Planck Institute for Biological Cybernetics}, event_place = {Hamburg, Germany}, event_name = {9th International Multisensory Research Forum (IMRF 2008)}, language = {en}, author = {Noppeney, U and Ostwald, D and Werner, S and Kleiner, M} } @Conference { 5266, title = {How priming enables us to understand speech in an impoverished context}, journal = {NeuroImage}, year = {2008}, month = {6}, volume = {41}, number = {Supplement 1}, pages = {S156}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Melbourne, Australia}, event_name = {14th Annual Meeting of the Organization for Human Brain Mapping (HBM 2008)}, DOI = {10.1016/j.neuroimage.2008.04.008}, author = {T{\"u}nnerhoff, J and Noppeney, U} } @Conference { 5079, title = {Audio-visual interactions in perception and response selection}, journal = {Experimentelle Psychologie}, year = {2008}, month = {3}, volume = {50}, abstract = {Both physical and physiological transmission times can differ between audition and vision. Under certain conditions, the brain reduces perceived asynchrony by adapting to this temporal discrepancy. In two experiments we investigated whether this recalibration is specific to auditory and visual stimuli, or whether other modality combinations (audiotactile, visuotactile) are affected, as well. We presented asynchronous audiovisual signals, with either auditory leading or visual leading. Then, using temporal order judgments we measured observers point of subjective simultaneity for three modality combinations. Results indicate an adjustment of perceived simultaneity for the audiovisual and the visuotactile modality pairs. We conclude that audiovisual adaptation is the result of a change of processing latencies of visual events. In a second experiment, we corroborate this finding. We demonstrate that reaction times to visual signals, but not to tactile or auditory signals, change as a result of audiovisual recalibration.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://www.teap.de/index.php/teap/marburg2008}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Marburg, Germany}, event_name = {50. Tagung Experimentell Arbeitender Psychologen (TeaP 2008)}, author = {Werner, S and Noppeney, U} } @Article { 4473, title = {The effects of visual deprivation on functional and structural organization of the human brain}, journal = {Neuroscience and Biobehavioral Reviews}, year = {2007}, month = {5}, volume = {31}, number = {8}, pages = {1169-1180}, abstract = {Early onset blindness allows one to investigate how the human brain adapts to sensory experience in infancy and early childhood. Over the past decade, lesion, functional and structural imaging studies have accumulated evidence that severe perturbations to visual experience alter the functional and structural organization of the human brain. Visual deprivation can induce plastic changes not only in the visual system, but also in the remaining intact sensory–motor system, secondary to altered experience using these spared modalities. In particular, occipital, usually visual, areas are reorganized and recruited by the remaining senses and higher cognitive tasks primarily through cortico-cortical connectivity. Importantly, these plastic changes vary as a function of timing and are most pronounced in early onset blindness. Thus, sensory experience shapes functional and structural brain organization during sensitive periods in neurodevelopment.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science?_ob=MImg\&_imagekey=B6T0J-4NR18D1-1-1\&_cdi=4864\&_user=29041\&_orig=search\&_coverDate=05\%2F13\%2F2007\&_sk=999999999\&view=c\&wchp=dGLzVlz-zSkzS\&md5=270478a8e6d61d1e25db697f0488522a\&ie=}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1016/j.neubiorev.2007.04.012}, author = {Noppeney, U} } @Article { 4232, title = {Temporal lobe lesions and semantic impairment: A comparison of Herpes Simplex Virus Encephalitis and Semantic Dementia}, journal = {Brain}, year = {2007}, month = {1}, volume = {130}, number = {4}, pages = {1138-1147}, abstract = {Both herpes simplex virus encephalitis (HSVE) and semantic dementia (SD) typically affect anterior temporal lobe structures. Using voxel-based morphometry (VBM), this study compared the structural damage in four HSVE patients having a semantic deficit particularly affecting knowledge of living things and six SD patients with semantic impairment across all categories tested. Each patient was assessed relative to a group of control subjects. In both patient groups, left anterior temporal damage extended into the amygdala. In patients with HSVE, extensive grey matter loss was observed predominantly in the medial parts of the anterior temporal cortices bilaterally in SD patients the abnormalities extended more laterally and posteriorly in either the left, right or both temporal lobes. Based on a lesion deficit rationale and converging results from several other sources of evidence, we suggest that (i) antero-medial temporal cortex may be important for processing and differentiating between concepts that are ‘tig htly packed’ in semantic space, such as living things, whereas (ii) inferolateral temporal cortex may play a more general role within the semantic system.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://brain.oxfordjournals.org/cgi/reprint/130/4/1138}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1093/brain/awl344}, author = {Noppeney, U and Patterson, K and Tyler, LK and Moss, H and Stamatakis, EA and Bright, P and Mummery, C and Price, CJ} } @Inbook { 3838, title = {The neural systems processing tool and action semantics}, year = {2007}, pages = {182-204}, abstract = {This chapter discusses the contributions of functional imaging to our understanding of how action and tool concepts are represented and processed in the human brain. Section 7.1. introduces cognitive models of semantic organization. Section 7.2. provides a brief overview of functional imaging approaches to identify brain regions that have specialized for processing action and tool representations. Section 7.3. discusses the relationship between the visuomotor system and semantic processing of actions. Section 7.4. investigates the effects of action type and visual experience on action-selective responses. Section 7.5. characterizes the neural systems engaged in tool processing and how they are modulated by task and stimulus modality. Section 7.6. delineates future directions that may enable us to characterize the neural mechanisms that mediate tool and action-selective brain responses.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.cambridge.org/gb/knowledge/isbn/item1172606/?site_locale=en_GB}, editor = {Hart, J. , M. A. Kraut}, publisher = {Cambridge University Press}, address = {Cambridge, UK}, booktitle = {The neural basis of semantic memory}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, ISBN = {0-521-84870-9}, author = {Noppeney, U} } @Poster { 4566, title = {Accumulation of object evidence from multiple senses}, journal = {NeuroImage}, year = {2007}, month = {6}, volume = {36}, number = {Supplement 1}, pages = {S109}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/Poster_HBM2007_noppeney_[0].pdf}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811907002789}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Chicago, IL, USA}, event_name = {13th Annual Meeting of the Organization for Human Brain Mapping (HBM 2007)}, language = {en}, DOI = {10.1016/j.neuroimage.2007.03.045}, author = {Noppeney, U and Ostwald, D and Kleiner, M and Werner, S} } @Poster { 4565, title = {Multi-sensory interactions in perceptual and response selection processes}, journal = {NeuroImage}, year = {2007}, month = {6}, volume = {36}, number = {Supplement 1}, pages = {S120}, url = {http://www.kyb.tuebingen.mpg.de/fileadmin/user_upload/files/publications/Poster_HBM2007_werner_[0].pdf}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811907002789}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Chicago, IL, USA}, event_name = {13th Annual Meeting of the Organization for Human Brain Mapping (HBM 2007)}, language = {en}, DOI = {10.1016/j.neuroimage.2007.03.045}, author = {Werner, S and Noppeney, U} } @Poster { 4607, title = {Neural systems involved in visual-tactile integration of shape information}, year = {2007}, month = {6}, volume = {2007}, pages = {3}, abstract = {The brain integrates multisensory information to create a coherent and more reliable perceptual estimate of the environment. This multisensory estimate is a linear combination of the individual unimodal estimates that are weighted by their relative reliabilities (e.g., Ernst and Banks, Nature, 2002). Here we explored the neural substrates underlying visual-tactile integration in shape processing. To identify multisensory integration sites, we correlated behavioural data with neural activity evoked by multisensory integration. Observers were presented with elliptical shapes that they could see and/or touch. Observers’ task was to judge the shape of the ellipse. Introducing conflicts between seen and felt shape allowed us to examine whether participants relied more on visual or tactile information (relative weight of vision and touch). To manipulate the weight attributed to vision, we degraded visual information. We observed a decrease in visual weight when vision was degraded and thus became less reliable. Discrimination performance increased when both modalities were presented together, indicating that visual and tactile shape information is indeed fused. BOLD response bilaterally in the anterior IPS is modulated by visual input. Change in BOLD signal these areas correlates with cue weights, suggesting that this activity reflects the relative weighting of vision and touch.}, url = {http://www.kyb.tuebingen.mpg.defileadmin/user_upload/files/publications/Psychologie_und_Gehirn_2007-Helbig.pdf}, department = {Department B{\"u}lthoff}, department2 = {Research Group Ernst}, department3 = {Research Group Noppeney}, web_url = {https://eldorado.tu-dortmund.de/handle/2003/24421}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Dortmund, Germany}, event_name = {Fachtagung Psychologie und Gehirn 2007}, language = {en}, author = {Helbig, HB and Noppeney, U and Ernst, M} } @Poster { WrightMNVRGHP2007, title = {Selective activation around the left occipito-temporal sulcus for words relative to pictures: Individual variability or false positives?}, journal = {NeuroImage}, year = {2007}, month = {6}, volume = {36}, number = {Supplement 1}, pages = {S64}, department = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science/article/pii/S1053811907002789}, event_place = {Chicago, IL, USA}, event_name = {13th Annual Meeting of the Organization for Human Brain Mapping (HBM 2007)}, DOI = {10.1016/j.neuroimage.2007.03.045}, author = {Wright, ND and Mechelli, A and Noppeney, U and Veltman, DJ and Rombouts, SARB and Glensman, J and Haynes, J-D and Price, CJ} } @Poster { 4609, title = {Neural systems involved in visual-tactile integration of shape information}, year = {2007}, month = {1}, department = {Department B{\"u}lthoff}, department2 = {Research Group Ernst}, department3 = {Research Group Noppeney}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Klosters, Switzerland}, event_name = {42nd Winter Seminar: “Biophysical Chemistry, Molecular Biology and Cybernetics of Cell Functions}, language = {en}, author = {Helbig, H and Noppeney, U and Ernst, M} } @Conference { 5378, title = {Naturalistic, metaphoric and linguistic auditory-visual interactions}, year = {2007}, month = {11}, volume = {37}, number = {662.8}, abstract = {To form a coherent percept of our dynamic environment, the brain merges motion information from the auditory and visual senses. Yet, not only ‘naturalistic’ direction information of auditory motion, but also ‘metaphoric’ direction information of dynamic pitch has been shown to influence visual motion discrimination. Here, we systematically investigate the neural systems that mediate auditory influences on visual motion discrimination in naturalistic, metaphoric and linguistic contexts. In a visual selective attention paradigm, subjects discriminated the direction of visual motion at several levels of ambiguity, while ignoring a simultaneous auditory stimulus that could be either congruent, absent or incongruent. Audio-visual congruency was defined at the 1) naturalistic, 2) metaphoric and 3) linguistic levels using three classes of auditory stimuli: 1) MOTION: left vs. right moving white noise, 2) PITCH: rising vs. falling pitch and 3) SPEECH: spoken German words denoting directions e.g. ‘links’ vs. ‘rechts’. At the behavioral level, all three classes of auditory stimuli induced a directional bias. Furthermore, this bias was not significantly different across contexts. At the neural level, the auditory influence on visual motion processing was identified through (1) the interaction between visual ambiguity and audition (presence vs. absence) and (2) the incongruency effect, separately for MOTION, PITCH and SPEECH. A significant interaction was revealed for MOTION in left hMT+/V5 and for SPEECH in right intraparietal sulcus. An incongruency effect was only observed for SPEECH in the left superior temporal gyrus and right middle frontal gyrus. Direct comparisons across contexts confirmed this functional dissociation: The interaction effect gradually decreased in left hMT+/V5 for MOTION>PITCH>SPEECH and in right IPS for SPEECH>PITCH>MOTION. Our results suggest that audition can influence visual motion discrimination at the naturalistic, metaphoric and linguistic levels. Yet, even though the auditory bias was comparable across contexts, our functional imaging results suggest that they are mediated by different neural systems. While naturalistic influences emerge in motion processing areas, linguistic interactions are revealed primarily in higher-level fronto-parietal regions.}, department = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://www.sfn.org/am2007/}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {San Diego, CA, USA}, event_name = {37th Annual Meeting of the Society for Neuroscience (Neuroscience 2007)}, language = {en}, author = {Sadaghiani, S and Noppeney, U} } @Article { 4082, title = {Language control in the bilingual brain}, journal = {Science}, year = {2006}, month = {6}, volume = {312}, number = {5779}, pages = {1537-1540}, abstract = {How does the bilingual brain distinguish and control which language is in use? Previous functional imaging experiments have not been able to answer this question because proficient bilinguals activate the same brain regions irrespective of the language being tested. Here, we reveal that neuronal responses within the left caudate are sensitive to changes in the language or the meaning of words. By demonstrating this effect in populations of German-English and Japanese-English bilinguals, we suggest that the left caudate plays a universal role in monitoring and controlling the language in use.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencemag.org/cgi/reprint/312/5779/1537.pdf}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1126/science.1127761}, author = {Crinion, J and Turner, R and Grogan, A and Hanakawa, T and Noppeney, U and Devlin, J and Aso, T and Urayama, S and Fukuyama, H and Stockton, K and Usui, K and Green, DW and Price, CJ} } @Article { 3689, title = {Identification of degenerate neuronal systems based on intersubject variability}, journal = {Neuroimage}, year = {2006}, month = {4}, volume = {30}, number = {3}, pages = {885-890}, abstract = {Group studies implicitly assume that all subjects activate one common system to sustain a particular cognitive task. Intersubject variability is generally treated as well-behaved and uninteresting noise. However, intersubject variability might result from subjects engaging different degenerate neuronal systems that are each sufficient for task performance. This would produce a multimodal distribution of intersubject variability. We have explored this idea with the help of Gaussian Mixture Modeling and Bayesian model comparison procedures. We illustrate our approach using a crossmodal priming paradigm, in which subjects perform a semantic decision on environmental sounds or their spoken names that were preceded by a semantically congruent or incongruent picture or written name. All subjects consistently activated the superior temporal gyri bilaterally, the left fusiform gyrus and the inferior frontal sulcus. Comparing a One and Two Gaussian Mixture Model of the unexplained residuals provided very strong eviden ce for two groups with distinct activation patterns: 6 subjects exhibited additional activations in the superior temporal sulci bilaterally, the right superior frontal and central sulcus. 11 subjects showed increased activation in the striate and the right inferior parietal cortex. These results suggest that semantic decisions on auditory–visual compound stimuli might be accomplished by two overlapping degenerate neuronal systems.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www.sciencedirect.com/science?_ob=MImg\&_imagekey=B6WNP-4HM7S0B-7-1\&_cdi=6968\&_user=29041\&_orig=browse\&_coverDate=04\%2F15\%2F2006\&_sk=999699996\&view=c\&wchp=dGLzVzz-zSkWW\&md5=a71a}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1016/j.neuroimage.2005.10.010}, author = {Noppeney, U and Penny, WD and Price, CJ and Flandin, G and Friston, KJ} } @Article { 3839, title = {Two approaches to repetition suppression}, journal = {Human Brain Mapping}, year = {2006}, month = {3}, volume = {27}, number = {5}, pages = {411-416}, abstract = {Repetition suppression refers to the phenomenon that prior processing of stimuli (or stimulus attributes) decreases activation elicited by processing subsequent stimuli with identical attributes. We present two complementary approaches to identify regions that show repetition suppression for subsequent sentences with either identical: (1) sentence forms or (2) speakers. The first categorical approach simply compares sentences that are presented in Same and Different blocks. The second factorial approach operationally defines repetition suppression as decreased activation for the subsequent Same stimulus relative to its preceding sentence. To account for nonspecific time confounds, this approach tests for a repetition \(\times\) condition (Same or Different) interaction. Surprisingly, the two approaches revealed different results: Only the categorical analysis detected sentence repetition effects in multiple regions within a bilateral frontotemporal system that has previously been implicated in sentence processing. Th ese discrepancies might be due to the different efficiencies with which the particular contrasts were estimated or spurious differences in stimuli or attentional set that could not be entirely controlled within a single subject. Finally, we combined the two approaches in a [global null] conjunction analysis.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://www3.interscience.wiley.com/cgi-bin/fulltext/112550800/PDFSTART}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, DOI = {10.1002/hbm.20242}, author = {Noppeney, U and Penny, WD} } @Inbook { 3710, title = {Functional neuroimaging of neuropsychologically impaired patients}, year = {2006}, month = {5}, pages = {455-480}, department = {Research Group Noppeney}, web_url = {http://cognet.mit.edu/library/erefs/cabeza/}, editor = {Cabeza, R. , A. Kingstone}, publisher = {MIT Press}, address = {Cambridge, MA, USA}, booktitle = {Handbook of Functional Neuroimaging of Cognition}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, language = {en}, author = {Price, CJ and Noppeney, U and Friston, KJ} } @Poster { SarkheilVBN2006, title = {Repetition priming in 3-D form and motion recognition}, journal = {Perception}, year = {2006}, month = {8}, volume = {35}, number = {ECVP Abstract Supplement}, pages = {35}, abstract = {Behavioural studies have highlighted the importance of dynamic information for object recognition: Object motion provides additional views and image features that may facilitate the extraction of 3-D shape. However, even the direction of in-depth rotation that controls for shape and view information affects recognition performance. Here, we used a priming paradigm to investigate the effects of motion direction and form as well as their interaction during dynamic object recognition. Furthermore, two task-contexts were used to investigate the effects of top - down modulation on behavioural priming effects. For these contexts, subjects responded on the basis of object form or motion. Subjects were presented with pairs of successive objects rotating in depth. They performed a two-alternative forced choice form or motion categorisation to the second object. The conditions conformed to a 2 \(\times\) 2 \(\times\) 2 factorial design manipulating (i) object form (same/different pairs), (ii) in-depth rotation (same/different pairs), and (iii) task (motion/form). We observed that form and motion priming effects interacted and were enhanced in congruent task context. These findings suggest that dynamic 3-D object recognition is accomplished through interaction of form and motion information. Furthermore, both form and motion priming are influenced by task requirements. Future fMRI studies will investigate these effects at the neuronal level.}, department = {Department B{\"u}lthoff}, department2 = {Department Logothetis}, department3 = {Research Group Noppeney}, web_url = {http://www.perceptionweb.com/abstract.cgi?id=v060400}, event_place = {St. Petersburg}, event_name = {29th European Conference on Visual Perception}, author = {Sarkheil, P and Vuong, QC and B{\"u}lthoff, HH and Noppeney, U} } @Poster { 4342, title = {Audio-visual integration during multisensory object categorization}, year = {2006}, month = {6}, volume = {7}, pages = {26}, abstract = {Tools or musical instruments are characterized by their form and sound. We investigated audio-visual integration during semantic categorization by presenting pictures and sounds of objects separately or together and manipulating the degree of information content. The 3 x 6 factorial design manipulated (1) auditory information (sound, noise, silence) and (2) visual information (6 levels of image degradation). The visual information was degraded by manipulating the amount of phase scrambling of the image (0\%, 20\%, 40\%, 60\%, 80\%, 100\%). Subjects categorized stimuli as musical instruments or tools. In terms of accuracy and reaction times (RT), we found significant main effects of (1) visual and (2) auditory information and (3) an interaction between the two factors. The interaction was primarily due to an increased facilitatory effect of sound for the 80\% degradation level. Consistently across the first 5 levels of visual degradation, we observed RT improvements for the sound-visual relative to the noise- or sile nce-visual conditions. Corresponding RT distributions significantly violated the so-called race model inequality across the first 5 percentiles of their cumulative density functions (even when controlling for low-level audio-visual interactions). These results suggest that redundant structural and semantic information is not independently processed but integrated during semantic categorization.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, web_url = {http://imrf.mcmaster.ca/IMRF/2006/viewabstract.php?id=124}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Dublin, Ireland}, event_name = {7th International Multisensory Research Forum (IMRF 2006)}, language = {en}, author = {Werner, S and Noppeney, U} } @Conference { 4343, title = {Higher-level audio-visual integration in human superior temporal sulcus}, year = {2006}, month = {11}, volume = {7}, pages = {3}, abstract = {Most objects and events can be detected by more than one sensory system. Thus, to form a coherent percept of the environment the brain has to combine information from different senses. Multisensory integration of visual and auditory information offers numerous benefits for the accuracy or completeness of the perception and seems to depend on neural substrates that can be found at various levels of the cortical processing hierarchy. The objective of the current human fMRI study was to identify and characterize brain regions mediating the integration of invariant higher-order audio-visual features that specify complex natural objects. We presented subjects with tools and musical instruments as pictures and sounds whilst manipulating their relative informativeness with respect to object recognition and controlling for their low-level features. Subjects were engaged in a categorization or target detection task. Across both tasks, we show higher-level audio-visual interactions in posterior, middle and anterior por tions of the superior temporal sulcus (STS), with those regions obeying the law of inverse effectiveness: integration effects were larger for less effective stimulus configurations. We demonstrate that these findings parallel indices of multisensory enhancement on the behavioral level. These results confirm the role of STS as a major convergence site of auditory and visual information and show that invariant higher-order audio-visual features are integrated within higher association areas of the brain.}, department = {Department B{\"u}lthoff}, department2 = {Research Group Noppeney}, talk_type = {Abstract Talk}, web_url = {http://www.neuroschool-tuebingen-nena.de/index.php?id=284}, institute = {Biologische Kybernetik}, organization = {Max-Planck-Gesellschaft}, event_place = {Oberjoch, Germany}, event_name = {7th Conference of the Junior Neuroscientists of Tuebingen (NeNa 2006)}, language = {en}, author = {Werner, S} }