BrainComputation.bib

@article{RN3403,
   author = {Albus, J. S.},
   title = {A theory of cerebellar function},
   journal = {Mathematical Bioscience},
   volume = {10},
   pages = {25-61},
   DOI = {10.1016/0025-5564(71)90051-4},
   year = {1971},
   type = {Journal Article}
}

@article{RN6672,
   author = {Amari, Shun-ichi},
   title = {Natural Gradient Works Efficiently in Learning},
   journal = {Neural Computation},
   volume = {10},
   number = {2},
   pages = {251-276},
   ISSN = {0899-7667
1530-888X},
   DOI = {10.1162/089976698300017746},
   year = {1998},
   type = {Journal Article}
}

@article{RN3991,
   author = {Aston-Jones, G. and Cohen, J. D.},
   title = {An integrative theory of locus coeruleus-norepinephrine function: adaptive gain and optimal performance},
   journal = {Annual Reviews in Neuroscience},
   volume = {28},
   pages = {403-50},
   note = {0147-006X (Print)
Journal Article
Research Support, N.I.H., Extramural
Research Support, U.S. Gov't, Non-P.H.S.
Research Support, U.S. Gov't, P.H.S.
Review},
   abstract = {Historically, the locus coeruleus-norepinephrine (LC-NE) system has been implicated in arousal, but recent findings suggest that this system plays a more complex and specific role in the control of behavior than investigators previously thought. We review neurophysiological and modeling studies in monkey that support a new theory of LC-NE function. LC neurons exhibit two modes of activity, phasic and tonic. Phasic LC activation is driven by the outcome of task-related decision processes and is proposed to facilitate ensuing behaviors and to help optimize task performance (exploitation). When utility in the task wanes, LC neurons exhibit a tonic activity mode, associated with disengagement from the current task and a search for alternative behaviors (exploration). Monkey LC receives prominent, direct inputs from the anterior cingulate (ACC) and orbitofrontal cortices (OFC), both of which are thought to monitor task-related utility. We propose that these frontal areas produce the above patterns of LC activity to optimize utility on both short and long timescales.},
   keywords = {Action Potentials/physiology
Adaptation, Physiological/*physiology
Animals
Brain Mapping
Cognition/physiology
Humans
Locus Coeruleus/*physiology
Neural Networks (Computer)
Norepinephrine/*physiology
*Systems Integration
Time Factors},
   DOI = {10.1146/annurev.neuro.28.061604.135709},
   url = {http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&dopt=Citation&list_uids=16022602 },
   year = {2005},
   type = {Journal Article}
}

@article{RN133,
   author = {Barraclough, D. J. and Conroy, M. L. and Lee, D.},
   title = {Prefrontal cortex and decision making in a mixed-strategy game},
   journal = {Nature Neuroscience},
   volume = {7},
   number = {4},
   pages = {404-10},
   note = {Barraclough, Dominic J
Conroy, Michelle L
Lee, Daeyeol
eng
EY01319/EY/NEI NIH HHS/
NS44270/NS/NINDS NIH HHS/
Nat Neurosci. 2004 Apr;7(4):404-10. Epub 2004 Mar 7.},
   abstract = {In a multi-agent environment, where the outcomes of one's actions change dynamically because they are related to the behavior of other beings, it becomes difficult to make an optimal decision about how to act. Although game theory provides normative solutions for decision making in groups, how such decision-making strategies are altered by experience is poorly understood. These adaptive processes might resemble reinforcement learning algorithms, which provide a general framework for finding optimal strategies in a dynamic environment. Here we investigated the role of prefrontal cortex (PFC) in dynamic decision making in monkeys. As in reinforcement learning, the animal's choice during a competitive game was biased by its choice and reward history, as well as by the strategies of its opponent. Furthermore, neurons in the dorsolateral prefrontal cortex (DLPFC) encoded the animal's past decisions and payoffs, as well as the conjunction between the two, providing signals necessary to update the estimates of expected reward. Thus, PFC might have a key role in optimizing decision-making strategies.},
   keywords = {Animals
Choice Behavior/*physiology
Electrophysiology
Evoked Potentials/physiology
*Game Theory
Macaca mulatta
Male
Mental Processes/*physiology
Neurons/physiology
Practice (Psychology)
Prefrontal Cortex/cytology/*physiology
*Probability Learning},
   ISSN = {1097-6256 (Print)
1097-6256 (Linking)},
   DOI = {10.1038/nn1209},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/15004564},
   year = {2004},
   type = {Journal Article}
}

@article{RN136,
   author = {Barto, A. G. and Sutton, R. S. and Andersen, C. W.},
   title = {Neuronlike adaptive elements that can solve difficult learning control problems},
   journal = {IEEE Transactions on Systems, Man, and Cybernetics},
   volume = {13},
   number = {5},
   pages = {834-846},
   DOI = {10.1109/TSMC.1983.6313077},
   year = {1983},
   type = {Journal Article}
}

@article{RN183,
   author = {Bell, A. J. and Sejnowski, T. J.},
   title = {An information-maxization approach to blind separation and blind deconvolution},
   journal = {Neural Computation},
   volume = {7},
   pages = {1129-1159},
   DOI = {10.1162/neco.1995.7.6.1129 },
   year = {1995},
   type = {Journal Article}
}

@article{RN184,
   author = {Bell, Anthony J. and Sejnowski, Terrence J.},
   title = {The “independent components” of natural scenes are edge filters},
   journal = {Vision Research},
   volume = {37},
   number = {23},
   pages = {3327-3338},
   ISSN = {00426989},
   DOI = {10.1016/s0042-6989(97)00121-1},
   year = {1997},
   type = {Journal Article}
}

@article{RN6732,
   author = {Bengio, Y. and Courville, A. and Vincent, P.},
   title = {Representation learning: a review and new perspectives},
   journal = {IEEE Trans Pattern Anal Mach Intell},
   volume = {35},
   number = {8},
   pages = {1798-828},
   note = {Bengio, Yoshua
Courville, Aaron
Vincent, Pascal
eng
Research Support, Non-U.S. Gov't
Review
IEEE Trans Pattern Anal Mach Intell. 2013 Aug;35(8):1798-828. doi: 10.1109/TPAMI.2013.50.},
   abstract = {The success of machine learning algorithms generally depends on data representation, and we hypothesize that this is because different representations can entangle and hide more or less the different explanatory factors of variation behind the data. Although specific domain knowledge can be used to help design representations, learning with generic priors can also be used, and the quest for AI is motivating the design of more powerful representation-learning algorithms implementing such priors. This paper reviews recent work in the area of unsupervised feature learning and deep learning, covering advances in probabilistic models, autoencoders, manifold learning, and deep networks. This motivates longer term unanswered questions about the appropriate objectives for learning good representations, for computing representations (i.e., inference), and the geometrical connections between representation learning, density estimation, and manifold learning.},
   keywords = {Algorithms
Artificial Intelligence/*trends
Humans
Neural Networks (Computer)},
   ISSN = {1939-3539 (Electronic)
0098-5589 (Linking)},
   DOI = {10.1109/TPAMI.2013.50},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/23787338},
   year = {2013},
   type = {Journal Article}
}

@article{RN223,
   author = {Bialek, W. and Rieke, F. and de Ruyter van Steveninck, R. R. and Warland, D.},
   title = {Reading a neural code},
   journal = {Science},
   volume = {252},
   number = {5014},
   pages = {1854-7},
   note = {Bialek, W
Rieke, F
de Ruyter van Steveninck, R R
Warland, D
eng
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Research Support, U.S. Gov't, P.H.S.
1991/06/28
Science. 1991 Jun 28;252(5014):1854-7. doi: 10.1126/science.2063199.},
   abstract = {Traditional approaches to neural coding characterize the encoding of known stimuli in average neural responses. Organisms face nearly the opposite task--extracting information about an unknown time-dependent stimulus from short segments of a spike train. Here the neural code was characterized from the point of view of the organism, culminating in algorithms for real-time stimulus estimation based on a single example of the spike train. These methods were applied to an identified movement-sensitive neuron in the fly visual system. Such decoding experiments determined the effective noise level and fault tolerance of neural computation, and the structure of the decoding algorithms suggested a simple model for real-time analog signal processing with spiking neurons.},
   keywords = {Algorithms
Animals
Diptera
Mathematics
*Models, Neurological
Neurons/*physiology
Neurons, Afferent/*physiology
Photoreceptor Cells/physiology
Visual Perception},
   ISSN = {0036-8075 (Print)
0036-8075 (Linking)},
   DOI = {10.1126/science.2063199},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/2063199},
   year = {1991},
   type = {Journal Article}
}

@book{RN7347,
   author = {Bishop, Christopher M.},
   title = {Pattern Recognition and Machine Learning},
   publisher = {Springer},
   address = {New York},
   ISBN = {978-0-387-31073-2},
   url = {https://www.microsoft.com/en-us/research/people/cmbishop/prml-book/},
   year = {2006},
   type = {Book}
}

@article{RN5201,
   author = {Bogacz, R.},
   title = {A tutorial on the free-energy framework for modelling perception and learning},
   journal = {Journal of Mathematical Psychology},
   volume = {76},
   number = {Pt B},
   pages = {198-211},
   note = {Bogacz, Rafal
eng
2017/03/17
J Math Psychol. 2017 Feb;76(Pt B):198-211. doi: 10.1016/j.jmp.2015.11.003.},
   abstract = {This paper provides an easy to follow tutorial on the free-energy framework for modelling perception developed by Friston, which extends the predictive coding model of Rao and Ballard. These models assume that the sensory cortex infers the most likely values of attributes or features of sensory stimuli from the noisy inputs encoding the stimuli. Remarkably, these models describe how this inference could be implemented in a network of very simple computational elements, suggesting that this inference could be performed by biological networks of neurons. Furthermore, learning about the parameters describing the features and their uncertainty is implemented in these models by simple rules of synaptic plasticity based on Hebbian learning. This tutorial introduces the free-energy framework using very simple examples, and provides step-by-step derivations of the model. It also discusses in more detail how the model could be implemented in biological neural circuits. In particular, it presents an extended version of the model in which the neurons only sum their inputs, and synaptic plasticity only depends on activity of pre-synaptic and post-synaptic neurons.},
   ISSN = {0022-2496 (Print)
0022-2496 (Linking)},
   DOI = {10.1016/j.jmp.2015.11.003},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/28298703},
   year = {2017},
   type = {Journal Article}
}

@article{RN11965,
   author = {Bogacz, R.},
   title = {Dopamine role in learning and action inference},
   journal = {Elife},
   volume = {9},
   note = {Bogacz, Rafal
eng
MC_UU_00003/1/MRC_/Medical Research Council/United Kingdom
MC_UU_12024/5/MRC_/Medical Research Council/United Kingdom
BB/S006338/1/BB_/Biotechnology and Biological Sciences Research Council/United Kingdom
Research Support, Non-U.S. Gov't
England
2020/07/08
Elife. 2020 Jul 7;9:e53262. doi: 10.7554/eLife.53262.},
   abstract = {This paper describes a framework for modelling dopamine function in the mammalian brain. It proposes that both learning and action planning involve processes minimizing prediction errors encoded by dopaminergic neurons. In this framework, dopaminergic neurons projecting to different parts of the striatum encode errors in predictions made by the corresponding systems within the basal ganglia. The dopaminergic neurons encode differences between rewards and expectations in the goal-directed system, and differences between the chosen and habitual actions in the habit system. These prediction errors trigger learning about rewards and habit formation, respectively. Additionally, dopaminergic neurons in the goal-directed system play a key role in action planning: They compute the difference between a desired reward and the reward expected from the current motor plan, and they facilitate action planning until this difference diminishes. Presented models account for dopaminergic responses during movements, effects of dopamine depletion on behaviour, and make several experimental predictions.
In the brain, chemicals such as dopamine allow nerve cells to 'talk' to each other and to relay information from and to the environment. Dopamine, in particular, is released when pleasant surprises are experienced: this helps the organism to learn about the consequences of certain actions. If a new flavour of ice-cream tastes better than expected, for example, the release of dopamine tells the brain that this flavour is worth choosing again. However, dopamine has an additional role in controlling movement. When the cells that produce dopamine die, for instance in Parkinson's disease, individuals may find it difficult to initiate deliberate movements. Here, Rafal Bogacz aimed to develop a comprehensive framework that could reconcile the two seemingly unrelated roles played by dopamine. The new theory proposes that dopamine is released when an outcome differs from expectations, which helps the organism to adjust and minimise these differences. In the ice-cream example, the difference is between how good the treat is expected to taste, and how tasty it really is. By learning to select the same flavour repeatedly, the brain aligns expectation and the result of the choice. This ability would also apply when movements are planned. In this case, the brain compares the desired reward with the predicted results of the planned actions. For example, while planning to get a spoonful of ice-cream, the brain compares the pleasure expected from the movement that is currently planned, and the pleasure of eating a full spoon of the treat. If the two differ, for example because no movement has been planned yet, the brain releases dopamine to form a better version of the action plan. The theory was then tested using a computer simulation of nerve cells that release dopamine; this showed that the behaviour of the virtual cells closely matched that of their real-life counterparts. This work offers a comprehensive description of the fundamental role of dopamine in the brain. The model now needs to be verified through experiments on living nerve cells; ultimately, it could help doctors and researchers to develop better treatments for conditions such as Parkinson's disease or ADHD, which are linked to a lack of dopamine.
eng},
   keywords = {Animals
Basal Ganglia/physiology
Corpus Striatum/*physiology
Dopamine/*metabolism
Dopaminergic Neurons/*physiology
Learning/*physiology
Mammals/*physiology
Models, Biological
*Reward
active inference
computational biology
dopamine
human
mouse
neuroscience
rat
reinforcement learning
rhesus macaque
systems biology},
   ISSN = {2050-084X (Electronic)
2050-084X (Linking)},
   DOI = {10.7554/eLife.53262},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/32633715},
   year = {2020},
   type = {Journal Article}
}

@article{RN411,
   author = {Courville, A. C. and Daw, N. D. and Touretzky, D. S.},
   title = {Bayesian theories of conditioning in a changing world},
   journal = {Trends Cogn Sci},
   volume = {10},
   number = {7},
   pages = {294-300},
   note = {Courville, Aaron C
Daw, Nathaniel D
Touretzky, David S
eng
England
Trends Cogn Sci. 2006 Jul;10(7):294-300. Epub 2006 Jun 21.},
   abstract = {The recent flowering of Bayesian approaches invites the re-examination of classic issues in behavior, even in areas as venerable as Pavlovian conditioning. A statistical account can offer a new, principled interpretation of behavior, and previous experiments and theories can inform many unexplored aspects of the Bayesian enterprise. Here we consider one such issue: the finding that surprising events provoke animals to learn faster. We suggest that, in a statistical account of conditioning, surprise signals change and therefore uncertainty and the need for new learning. We discuss inference in a world that changes and show how experimental results involving surprise can be interpreted from this perspective, and also how, thus understood, these phenomena help constrain statistical theories of animal and human learning.},
   keywords = {Animals
Association Learning
*Bayes Theorem
*Behavior, Animal
*Conditioning, Classical
Extinction, Psychological
Humans
Inhibition (Psychology)
Problem Solving
Reinforcement Schedule
*Social Change
Social Environment
Stochastic Processes
Uncertainty},
   ISSN = {1364-6613 (Print)
1364-6613 (Linking)},
   DOI = {10.1016/j.tics.2006.05.004},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/16793323},
   year = {2006},
   type = {Journal Article}
}

@article{RN460,
   author = {Daw, N. D. and Gershman, S. J. and Seymour, B. and Dayan, P. and Dolan, R. J.},
   title = {Model-based influences on humans' choices and striatal prediction errors},
   journal = {Neuron},
   volume = {69},
   number = {6},
   pages = {1204-15},
   note = {Daw, Nathaniel D
Gershman, Samuel J
Seymour, Ben
Dayan, Peter
Dolan, Raymond J
eng
1R01MH087882-01/MH/NIMH NIH HHS/
R01 MH087882-02/MH/NIMH NIH HHS/
Wellcome Trust/United Kingdom
Neuron. 2011 Mar 24;69(6):1204-15. doi: 10.1016/j.neuron.2011.02.027.},
   abstract = {The mesostriatal dopamine system is prominently implicated in model-free reinforcement learning, with fMRI BOLD signals in ventral striatum notably covarying with model-free prediction errors. However, latent learning and devaluation studies show that behavior also shows hallmarks of model-based planning, and the interaction between model-based and model-free values, prediction errors, and preferences is underexplored. We designed a multistep decision task in which model-based and model-free influences on human choice behavior could be distinguished. By showing that choices reflected both influences we could then test the purity of the ventral striatal BOLD signal as a model-free report. Contrary to expectations, the signal reflected both model-free and model-based predictions in proportions matching those that best explained choice behavior. These results challenge the notion of a separate model-free learner and suggest a more integrated computational architecture for high-level human decision-making.},
   keywords = {Adult
Basal Ganglia/*physiology
Brain Mapping
Choice Behavior/*physiology
Dopamine/*metabolism
Female
Humans
Logistic Models
Magnetic Resonance Imaging
Male
Models, Neurological
Neurons/physiology
Neuropsychological Tests
*Reinforcement (Psychology)},
   ISSN = {1097-4199 (Electronic)
0896-6273 (Linking)},
   DOI = {10.1016/j.neuron.2011.02.027},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/21435563},
   year = {2011},
   type = {Journal Article}
}

@book{RN7345,
   author = {Dayan, P and Abbott, L F},
   title = {Theoretical Neuroscience: Computation and Mathematical Modeling of Neural Systems},
   publisher = {MIT Press},
   url = {http://www.gatsby.ucl.ac.uk/~dayan/book/},
   year = {2001},
   type = {Book}
}

@techreport{RN5582,
   author = {Doersch, Carl},
   title = {Tutorial on variational autoencoders},
   year = {2016},
   type = {Report}
}

@article{RN2997,
   author = {Doya, K.},
   title = {What are the computations of the cerebellum, the basal ganglia, and the cerebral cortex},
   journal = {Neural Networks},
   volume = {12},
   pages = {961-974},
   note = {Neural Networks},
   keywords = {basal ganglia},
   DOI = {10.1016/S0893-6080(99)00046-5},
   year = {1999},
   type = {Journal Article}
}

@article{RN3028,
   author = {Doya, K.},
   title = {Complementary roles of basal ganglia and cerebellum in learning and motor control},
   journal = {Curr Opin Neurobiol},
   volume = {10},
   number = {6},
   pages = {732-9},
   note = {Doya, K
eng
Review
England
2001/03/10
Curr Opin Neurobiol. 2000 Dec;10(6):732-9. doi: 10.1016/s0959-4388(00)00153-7.},
   abstract = {The classical notion that the basal ganglia and the cerebellum are dedicated to motor control has been challenged by the accumulation of evidence revealing their involvement in non-motor, cognitive functions. From a computational viewpoint, it has been suggested that the cerebellum, the basal ganglia, and the cerebral cortex are specialized for different types of learning: namely, supervised learning, reinforcement learning and unsupervised learning, respectively. This idea of learning-oriented specialization is helpful in understanding the complementary roles of the basal ganglia and the cerebellum in motor control and cognitive functions.},
   keywords = {Animals
Basal Ganglia/cytology/*physiology
Cerebellum/cytology/*physiology
Humans
Learning/*physiology
Motor Neurons/physiology
Movement/*physiology},
   ISSN = {0959-4388 (Print)
0959-4388 (Linking)},
   DOI = {10.1016/s0959-4388(00)00153-7},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/11240282},
   year = {2000},
   type = {Journal Article}
}

@article{RN3152,
   author = {Doya, K.},
   title = {Metalearning and Neuromodulation},
   journal = {Neural Networks},
   volume = {15},
   pages = {495–506},
   note = {Neural Networks},
   DOI = {10.1016/S0893-6080(02)00044-8},
   year = {2002},
   type = {Journal Article}
}

@article{RN765,
   author = {Doya, K.},
   title = {Reinforcement learning: Computational theory and biological mechanisms},
   journal = {HFSP Journal},
   volume = {1},
   number = {1},
   pages = {30-40},
   note = {Doya, Kenji
eng
France
HFSP J. 2007 May;1(1):30-40. doi: 10.2976/1.2732246/10.2976/1. Epub 2007 May 8.},
   abstract = {Reinforcement learning is a computational framework for an active agent to learn behaviors on the basis of a scalar reward signal. The agent can be an animal, a human, or an artificial system such as a robot or a computer program. The reward can be food, water, money, or whatever measure of the performance of the agent. The theory of reinforcement learning, which was developed in an artificial intelligence community with intuitions from animal learning theory, is now giving a coherent account on the function of the basal ganglia. It now serves as the "common language" in which biologists, engineers, and social scientists can exchange their problems and findings. This article reviews the basic theoretical framework of reinforcement learning and discusses its recent and future contributions toward the understanding of animal behaviors and human decision making.},
   ISSN = {1955-2068 (Print)
1955-205X (Linking)},
   DOI = {10.2976/1.2732246/10.2976/1},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/19404458},
   year = {2007},
   type = {Journal Article}
}

@article{RN768,
   author = {Doya, K.},
   title = {Modulators of decision making},
   journal = {Nature Neuroscience},
   volume = {11},
   number = {4},
   pages = {410-6},
   note = {Doya, Kenji
eng
Nat Neurosci. 2008 Apr;11(4):410-6. doi: 10.1038/nn2077.},
   abstract = {Human and animal decisions are modulated by a variety of environmental and intrinsic contexts. Here I consider computational factors that can affect decision making and review anatomical structures and neurochemical systems that are related to contextual modulation of decision making. Expectation of a high reward can motivate a subject to go for an action despite a large cost, a decision that is influenced by dopamine in the anterior cingulate cortex. Uncertainty of action outcomes can promote risk taking and exploratory choices, in which norepinephrine and the orbitofrontal cortex appear to be involved. Predictable environments should facilitate consideration of longer-delayed rewards, which depends on serotonin in the dorsal striatum and dorsal prefrontal cortex. This article aims to sort out factors that affect the process of decision making from the viewpoint of reinforcement learning theory and to bridge between such computational needs and their neurophysiological substrates.},
   keywords = {Algorithms
Animals
*Computational Biology
*Decision Making
Environment
Gyrus Cinguli/*physiology
Humans
*Models, Neurological
Models, Psychological
Neurotransmitter Agents/physiology
*Probability Learning
Reward},
   ISSN = {1097-6256 (Print)
1097-6256 (Linking)},
   DOI = {10.1038/nn2077},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/18368048},
   year = {2008},
   type = {Journal Article}
}

@book{RN3237,
   author = {Doya, K. and Ishii, S. and Pouget, A. and Rao, R. },
   title = {Bayesian Brain: Probabilistic Approach to Neural Coding and Learning},
   publisher = {MIT Press},
   year = {2007},
   type = {Book}
}

@article{RN9413,
   author = {Doya, Kenji and Miyazaki, Kayoko W. and Miyazaki, Katsuhiko},
   title = {Serotonergic modulation of cognitive computations},
   journal = {Current Opinion in Behavioral Sciences},
   volume = {38},
   pages = {116-123},
   ISSN = {23521546},
   DOI = {10.1016/j.cobeha.2021.02.003},
   year = {2021},
   type = {Journal Article}
}

@article{RN779,
   author = {Elfwing, S. and Uchibe, E. and Doya, K. and Christensen, H. I.},
   title = {Darwinian embodied evolution of the learning ability for survival},
   journal = {Adaptive Behavior},
   volume = {19},
   number = {2},
   pages = {101-120},
   ISSN = {1059-7123
1741-2633},
   DOI = {10.1177/1059712310397633},
   year = {2011},
   type = {Journal Article}
}

@article{RN12253,
   author = {Ellery, A.},
   title = {Tutorial Review of Bio-Inspired Approaches to Robotic Manipulation for Space Debris Salvage},
   journal = {Biomimetics (Basel)},
   volume = {5},
   number = {2},
   note = {Ellery, Alex
eng
Review
Switzerland
2020/05/16
Biomimetics (Basel). 2020 May 12;5(2):19. doi: 10.3390/biomimetics5020019.},
   abstract = {We present a comprehensive tutorial review that explores the application of bio-inspired approaches to robot control systems for grappling and manipulating a wide range of space debris targets. Current robot manipulator control systems exploit limited techniques which can be supplemented by additional bio-inspired methods to provide a robust suite of robot manipulation technologies. In doing so, we review bio-inspired control methods because this will be the key to enabling such capabilities. In particular, force feedback control may be supplemented with predictive forward models and software emulation of viscoelastic preflexive joint behaviour. This models human manipulation capabilities as implemented by the cerebellum and muscles/joints respectively. In effect, we are proposing a three-level control strategy based on biomimetic forward models for predictive estimation, traditional feedback control and biomimetic muscle-like preflexes. We place emphasis on bio-inspired forward modelling suggesting that all roads lead to this solution for robust and adaptive manipulator control. This promises robust and adaptive manipulation for complex tasks in salvaging space debris.},
   keywords = {cerebellum
on-orbit servicing
predictive forward models
preflex
space debris mitigation
space salvage
viscoelastic muscle},
   ISSN = {2313-7673 (Electronic)
2313-7673 (Linking)},
   DOI = {10.3390/biomimetics5020019},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/32408615},
   year = {2020},
   type = {Journal Article}
}

@article{RN12252,
   author = {Engel, A. K. and Fries, P. and Singer, W.},
   title = {Dynamic predictions: oscillations and synchrony in top-down processing},
   journal = {Nat Rev Neurosci},
   volume = {2},
   number = {10},
   pages = {704-16},
   note = {Engel, A K
Fries, P
Singer, W
eng
Review
England
2001/10/05
Nat Rev Neurosci. 2001 Oct;2(10):704-16. doi: 10.1038/35094565.},
   abstract = {Classical theories of sensory processing view the brain as a passive, stimulus-driven device. By contrast, more recent approaches emphasize the constructive nature of perception, viewing it as an active and highly selective process. Indeed, there is ample evidence that the processing of stimuli is controlled by top-down influences that strongly shape the intrinsic dynamics of thalamocortical networks and constantly create predictions about forthcoming sensory events. We discuss recent experiments indicating that such predictions might be embodied in the temporal structure of both stimulus-evoked and ongoing activity, and that synchronous oscillations are particularly important in this process. Coherence among subthreshold membrane potential fluctuations could be exploited to express selective functional relationships during states of expectancy or attention, and these dynamic patterns could allow the grouping and selection of distributed neuronal responses for further processing.},
   keywords = {Animals
Brain/*physiology
Haplorhini
Humans
Mental Processes
Models, Neurological
Motor Activity/physiology
Neurons/*physiology
Oscillometry
Pattern Recognition, Visual
Time Factors
Visual Cortex/physiology},
   ISSN = {1471-003X (Print)
1471-003X (Linking)},
   DOI = {10.1038/35094565},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/11584308},
   year = {2001},
   type = {Journal Article}
}

@article{RN4762,
   author = {Fermin, Alan and Yoshida, Takehiko and Ito, Makoto and Yoshimoto, Junichiro and Doya, Kenji},
   title = {Neural mechanisms for model-free and model-based reinforcement strategies in humans performing a multi-step navigation task},
   journal = {Neuroscience Research},
   volume = {68},
   pages = {E285-E286},
   ISSN = {0168-0102},
   DOI = {10.1016/j.neures.2010.07.1269},
   year = {2010},
   type = {Journal Article}
}

@techreport{RN7373,
   author = {Fernando, Chrisantha and Banarse, Dylan and Blundell, Charles and Zwols, Yori and Ha, David and Rusu, Andrei A. and Pritzel, Alexander and Wierstra, Daan},
   title = {Pathnet: Evolution channels gradient descent in super neural networks},
   DOI = {10.48550/arXiv.1701.08734},
   year = {2017},
   type = {Report}
}

@article{RN7478,
   author = {Friston, K.},
   title = {A theory of cortical responses},
   journal = {Philos Trans R Soc Lond B Biol Sci},
   volume = {360},
   number = {1456},
   pages = {815-36},
   note = {Friston, Karl
eng
Wellcome Trust/United Kingdom
Research Support, Non-U.S. Gov't
Review
England
Philos Trans R Soc Lond B Biol Sci. 2005 Apr 29;360(1456):815-36. doi: 10.1098/rstb.2005.1622.},
   abstract = {This article concerns the nature of evoked brain responses and the principles underlying their generation. We start with the premise that the sensory brain has evolved to represent or infer the causes of changes in its sensory inputs. The problem of inference is well formulated in statistical terms. The statistical fundaments of inference may therefore afford important constraints on neuronal implementation. By formulating the original ideas of Helmholtz on perception, in terms of modern-day statistical theories, one arrives at a model of perceptual inference and learning that can explain a remarkable range of neurobiological facts.It turns out that the problems of inferring the causes of sensory input (perceptual inference) and learning the relationship between input and cause (perceptual learning) can be resolved using exactly the same principle. Specifically, both inference and learning rest on minimizing the brain's free energy, as defined in statistical physics. Furthermore, inference and learning can proceed in a biologically plausible fashion. Cortical responses can be seen as the brain's attempt to minimize the free energy induced by a stimulus and thereby encode the most likely cause of that stimulus. Similarly, learning emerges from changes in synaptic efficacy that minimize the free energy, averaged over all stimuli encountered. The underlying scheme rests on empirical Bayes and hierarchical models of how sensory input is caused. The use of hierarchical models enables the brain to construct prior expectations in a dynamic and context-sensitive fashion. This scheme provides a principled way to understand many aspects of cortical organization and responses. The aim of this article is to encompass many apparently unrelated anatomical, physiological and psychophysical attributes of the brain within a single theoretical perspective. In terms of cortical architectures, the theoretical treatment predicts that sensory cortex should be arranged hierarchically, that connections should be reciprocal and that forward and backward connections should show a functional asymmetry (forward connections are driving, whereas backward connections are both driving and modulatory). In terms of synaptic physiology, it predicts associative plasticity and, for dynamic models, spike-timing-dependent plasticity. In terms of electrophysiology, it accounts for classical and extra classical receptive field effects and long-latency or endogenous components of evoked cortical responses. It predicts the attenuation of responses encoding prediction error with perceptual learning and explains many phenomena such as repetition suppression, mismatch negativity (MMN) and the P300 in electroencephalography. In psychophysical terms, it accounts for the behavioural correlates of these physiological phenomena, for example, priming and global precedence. The final focus of this article is on perceptual learning as measured with the MMN and the implications for empirical studies of coupling among cortical areas using evoked sensory responses.},
   keywords = {Biophysical Phenomena
Biophysics
Cerebral Cortex/*anatomy & histology/*physiology
Electrophysiology
Humans
Interneurons/cytology/physiology
Learning/*physiology
*Models, Neurological
*Models, Statistical
Perception/*physiology
Synapses/physiology},
   ISSN = {0962-8436 (Print)
0962-8436 (Linking)},
   DOI = {10.1098/rstb.2005.1622},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/15937014},
   year = {2005},
   type = {Journal Article}
}

@article{RN1008,
   author = {Friston, K.},
   title = {The free-energy principle: a unified brain theory?},
   journal = {Nat Rev Neurosci},
   volume = {11},
   number = {2},
   pages = {127-38},
   note = {Friston, Karl
eng
088130/Wellcome Trust/United Kingdom
Wellcome Trust/United Kingdom
England
Nat Rev Neurosci. 2010 Feb;11(2):127-38. doi: 10.1038/nrn2787. Epub 2010 Jan 13.},
   abstract = {A free-energy principle has been proposed recently that accounts for action, perception and learning. This Review looks at some key brain theories in the biological (for example, neural Darwinism) and physical (for example, information theory and optimal control theory) sciences from the free-energy perspective. Crucially, one key theme runs through each of these theories - optimization. Furthermore, if we look closely at what is optimized, the same quantity keeps emerging, namely value (expected reward, expected utility) or its complement, surprise (prediction error, expected cost). This is the quantity that is optimized under the free-energy principle, which suggests that several global brain theories might be unified within a free-energy framework.},
   keywords = {Animals
Brain/*physiology
Cognition/physiology
Humans
Learning/physiology
Nerve Net/*physiology
Perception/physiology
*Psychological Theory},
   ISSN = {1471-0048 (Electronic)
1471-003X (Linking)},
   DOI = {10.1038/nrn2787},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/20068583},
   year = {2010},
   type = {Journal Article}
}

@article{RN4155,
   author = {Fukushima, K.},
   title = {Neocognitron: a self organizing neural network model for a mechanism of pattern recognition unaffected by shift in position},
   journal = {Biol Cybern},
   volume = {36},
   number = {4},
   pages = {193-202},
   note = {Fukushima, K
eng
Germany
1980/01/01
Biol Cybern. 1980;36(4):193-202. doi: 10.1007/BF00344251.},
   abstract = {A neural network model for a mechanism of visual pattern recognition is proposed in this paper. The network is self-organized by "learning without a teacher", and acquires an ability to recognize stimulus patterns based on the geometrical similarity (Gestalt) of their shapes without affected by their positions. This network is given a nickname "neocognitron". After completion of self-organization, the network has a structure similar to the hierarchy model of the visual nervous system proposed by Hubel and Wiesel. The network consists of an input layer (photoreceptor array) followed by a cascade connection of a number of modular structures, each of which is composed of two layers of cells connected in a cascade. The first layer of each module consists of "S-cells", which show characteristics similar to simple cells or lower order hypercomplex cells, and the second layer consists of "C-cells" similar to complex cells or higher order hypercomplex cells. The afferent synapses to each S-cell have plasticity and are modifiable. The network has an ability of unsupervised learning: We do not need any "teacher" during the process of self-organization, and it is only needed to present a set of stimulus patterns repeatedly to the input layer of the network. The network has been simulated on a digital computer. After repetitive presentation of a set of stimulus patterns, each stimulus pattern has become to elicit an output only from one of the C-cells of the last layer, and conversely, this C-cell has become selectively responsive only to that stimulus pattern. That is, none of the C-cells of the last layer responds to more than one stimulus pattern. The response of the C-cells of the last layer is not affected by the pattern's position at all. Neither is it affected by a small change in shape nor in size of the stimulus pattern.},
   keywords = {Cognition/physiology
Computers
*Form Perception
Learning/*physiology
*Models, Neurological
Nerve Net/*physiology
*Nervous System Physiological Phenomena
*Pattern Recognition, Visual},
   ISSN = {0340-1200 (Print)
0340-1200 (Linking)},
   DOI = {10.1007/BF00344251},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/7370364},
   year = {1980},
   type = {Journal Article}
}

@article{RN5530,
   author = {Funamizu, A. and Kuhn, B. and Doya, K.},
   title = {Neural substrate of dynamic Bayesian inference in the cerebral cortex},
   journal = {Nature Neuroscience},
   volume = {19},
   number = {12},
   pages = {1682-1689},
   note = {Funamizu, Akihiro
Kuhn, Bernd
Doya, Kenji
eng
2016/11/01 06:00
Nat Neurosci. 2016 Dec;19(12):1682-1689. doi: 10.1038/nn.4390. Epub 2016 Sep 19.},
   abstract = {Dynamic Bayesian inference allows a system to infer the environmental state under conditions of limited sensory observation. Using a goal-reaching task, we found that posterior parietal cortex (PPC) and adjacent posteromedial cortex (PM) implemented the two fundamental features of dynamic Bayesian inference: prediction of hidden states using an internal state transition model and updating the prediction with new sensory evidence. We optically imaged the activity of neurons in mouse PPC and PM layers 2, 3 and 5 in an acoustic virtual-reality system. As mice approached a reward site, anticipatory licking increased even when sound cues were intermittently presented; this was disturbed by PPC silencing. Probabilistic population decoding revealed that neurons in PPC and PM represented goal distances during sound omission (prediction), particularly in PPC layers 3 and 5, and prediction improved with the observation of cue sounds (updating). Our results illustrate how cerebral cortex realizes mental simulation using an action-dependent dynamic model.},
   ISSN = {1546-1726 (Electronic)
1097-6256 (Linking)},
   DOI = {10.1038/nn.4390},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/27643432
http://www.nature.com/neuro/journal/v19/n12/pdf/nn.4390.pdf
http://www.nature.com/neuro/journal/v19/n12/full/nn.4390.html},
   year = {2016},
   type = {Journal Article}
}

@book{RN5898,
   author = {Gerstner, Wulfram and Kistler, Werner M. and Naud, Richard and Paninski, Liam},
   title = {Neuronal Dynamics: From Single Neurons to Networks and Models of Cognition},
   publisher = {Cambridge Universty Press},
   address = {Cambridge, UK},
   ISBN = {978-1-107-06083-8},
   url = {https://neuronaldynamics.epfl.ch},
   year = {2014},
   type = {Book}
}

@article{RN3430,
   author = {Ghahramani, Z. and Wolpert, D. M.},
   title = {Modular decomposition in visuomotor learning},
   journal = {Nature},
   volume = {386},
   pages = {392-395},
   DOI = {10.1038/386392a0},
   year = {1997},
   type = {Journal Article}
}

@article{RN1063,
   author = {Glascher, J. and Daw, N. and Dayan, P. and O'Doherty, J. P.},
   title = {States versus rewards: dissociable neural prediction error signals underlying model-based and model-free reinforcement learning},
   journal = {Neuron},
   volume = {66},
   number = {4},
   pages = {585-95},
   note = {Glascher, Jan
Daw, Nathaniel
Dayan, Peter
O'Doherty, John P
eng
R03 MH075763-02/MH/NIMH NIH HHS/
Neuron. 2010 May 27;66(4):585-95. doi: 10.1016/j.neuron.2010.04.016.},
   abstract = {Reinforcement learning (RL) uses sequential experience with situations ("states") and outcomes to assess actions. Whereas model-free RL uses this experience directly, in the form of a reward prediction error (RPE), model-based RL uses it indirectly, building a model of the state transition and outcome structure of the environment, and evaluating actions by searching this model. A state prediction error (SPE) plays a central role, reporting discrepancies between the current model and the observed state transitions. Using functional magnetic resonance imaging in humans solving a probabilistic Markov decision task, we found the neural signature of an SPE in the intraparietal sulcus and lateral prefrontal cortex, in addition to the previously well-characterized RPE in the ventral striatum. This finding supports the existence of two unique forms of learning signal in humans, which may form the basis of distinct computational strategies for guiding behavior.},
   keywords = {Adolescent
Adult
Choice Behavior/physiology
Female
Forecasting
Humans
Learning/physiology
Male
*Models, Neurological
Psychomotor Performance/*physiology
*Reinforcement (Psychology)
*Research Design
*Reward
Young Adult},
   ISSN = {1097-4199 (Electronic)
0896-6273 (Linking)},
   DOI = {10.1016/j.neuron.2010.04.016},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/20510862},
   year = {2010},
   type = {Journal Article}
}

@book{RN7326,
   author = {Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron},
   title = {Deep Learning},
   publisher = {MIT Press},
   year = {2016},
   type = {Book}
}

@article{RN1091,
   author = {Gray, Chrles M. and Koenig, Peter and Engel, Andreas K. and Singer, Wolf},
   title = {Oscillatory responses in cat visual cortex exhibit inter-columnar synchronication which reflects global stimulus properties},
   journal = {Nature},
   volume = {338},
   pages = {334-337},
   DOI = {10.1038/338334a0},
   year = {1989},
   type = {Journal Article}
}

@article{RN10280,
   author = {Grossman, C. D. and Bari, B. A. and Cohen, J. Y.},
   title = {Serotonin neurons modulate learning rate through uncertainty},
   journal = {Curr Biol},
   note = {Grossman, Cooper D
Bari, Bilal A
Cohen, Jeremiah Y
eng
England
Curr Biol. 2021 Dec 17. pii: S0960-9822(21)01682-1. doi: 10.1016/j.cub.2021.12.006.},
   abstract = {Regulating how fast to learn is critical for flexible behavior. Learning about the consequences of actions should be slow in stable environments, but accelerate when that environment changes. Recognizing stability and detecting change are difficult in environments with noisy relationships between actions and outcomes. Under these conditions, theories propose that uncertainty can be used to modulate learning rates ("meta-learning"). We show that mice behaving in a dynamic foraging task exhibit choice behavior that varied as a function of two forms of uncertainty estimated from a meta-learning model. The activity of dorsal raphe serotonin neurons tracked both types of uncertainty in the foraging task as well as in a dynamic Pavlovian task. Reversible inhibition of serotonin neurons in the foraging task reproduced changes in learning predicted by a simulated lesion of meta-learning in the model. We thus provide a quantitative link between serotonin neuron activity, learning, and decision making.},
   keywords = {decision making
dorsal raphe
learning
serotonin
uncertainty},
   ISSN = {1879-0445 (Electronic)
0960-9822 (Linking)},
   DOI = {10.1016/j.cub.2021.12.006},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/34936883},
   year = {2021},
   type = {Journal Article}
}

@article{RN6130,
   author = {Hassabis, D. and Kumaran, D. and Summerfield, C. and Botvinick, M.},
   title = {Neuroscience-inspired artificial intelligence},
   journal = {Neuron},
   volume = {95},
   number = {2},
   pages = {245-258},
   note = {Hassabis, Demis
Kumaran, Dharshan
Summerfield, Christopher
Botvinick, Matthew
eng
Review
Neuron. 2017 Jul 19;95(2):245-258. doi: 10.1016/j.neuron.2017.06.011.},
   abstract = {The fields of neuroscience and artificial intelligence (AI) have a long and intertwined history. In more recent times, however, communication and collaboration between the two fields has become less commonplace. In this article, we argue that better understanding biological brains could play a vital role in building intelligent machines. We survey historical interactions between the AI and neuroscience fields and emphasize current advances in AI that have been inspired by the study of neural computation in humans and other animals. We conclude by highlighting shared themes that may be key for advancing future research in both fields.},
   keywords = {artificial intelligence
brain
cognition
learning
neural network},
   ISSN = {1097-4199 (Electronic)
0896-6273 (Linking)},
   DOI = {10.1016/j.neuron.2017.06.011},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/28728020},
   year = {2017},
   type = {Journal Article}
}

@article{RN1162,
   author = {Hasselmo, M. E. and Sarter, M.},
   title = {Modes and models of forebrain cholinergic neuromodulation of cognition},
   journal = {Neuropsychopharmacology},
   volume = {36},
   number = {1},
   pages = {52-73},
   abstract = {As indicated by the profound cognitive impairments caused by cholinergic receptor antagonists, cholinergic neurotransmission has a vital role in cognitive function, specifically attention and memory encoding. Abnormally regulated cholinergic neurotransmission has been hypothesized to contribute to the cognitive symptoms of neuropsychiatric disorders. Loss of cholinergic neurons enhances the severity of the symptoms of dementia. Cholinergic receptor agonists and acetylcholinesterase inhibitors have been investigated for the treatment of cognitive dysfunction. Evidence from experiments using new techniques for measuring rapid changes in cholinergic neurotransmission provides a novel perspective on the cholinergic regulation of cognitive processes. This evidence indicates that changes in cholinergic modulation on a timescale of seconds is triggered by sensory input cues and serves to facilitate cue detection and attentional performance. Furthermore, the evidence indicates cholinergic induction of evoked intrinsic, persistent spiking mechanisms for active maintenance of sensory input, and planned responses. Models have been developed to describe the neuronal mechanisms underlying the transient modulation of cortical target circuits by cholinergic activity. These models postulate specific locations and roles of nicotinic and muscarinic acetylcholine receptors and that cholinergic neurotransmission is controlled in part by (cortical) target circuits. The available evidence and these models point to new principles governing the development of the next generation of cholinergic treatments for cognitive disorders.},
   keywords = {Acetylcholine/*physiology
Action Potentials/physiology
Animals
Attention/physiology
Cholinergic Fibers/*physiology
Cognition/*physiology
Humans
*Models, Neurological
Neural Pathways/anatomy & histology/chemistry/physiology
Neurons/physiology
Perception/physiology
Prosencephalon/anatomy & histology/chemistry/*physiology},
   ISSN = {1740-634X (Electronic)
0006-3223 (Linking)},
   DOI = {10.1038/npp.2010.104},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/20668433},
   year = {2011},
   type = {Journal Article}
}

@techreport{RN1204,
   author = {Hinton, Geoffrey},
   title = {A practical guide to training restricted Boltzmann machines},
   institution = {Department of Computer Science, University of Toronto},
   year = {2010},
   type = {Report}
}

@article{RN7348,
   author = {Hodgkin, A. L. and Huxley, A. F.},
   title = {A quantitative description of membrane current and its application to conduction and excitation in nerve},
   journal = {Journal of Physiology},
   volume = {117},
   pages = {500-544},
   DOI = {10.1113/jphysiol.1952.sp004764},
   year = {１９５２},
   type = {Journal Article}
}

@article{RN6770,
   author = {Horikawa, Tomoyasu and Kamitani, Yukiyasu},
   title = {Generic decoding of seen and imagined objects using hierarchical visual features},
   journal = {Nature Communications},
   volume = {8},
   ISSN = {2041-1723},
   DOI = {10.1038/ncomms15037},
   year = {2017},
   type = {Journal Article}
}

@article{RN4035,
   author = {Hubel, D. H. and Wiesel, T. N.},
   title = {Receptive fields of single neurones in the cat's striate cortex},
   journal = {Journal of Physiology},
   volume = {148},
   pages = {574-91},
   note = {HUBEL, D H
WIESEL, T N
eng
England
J Physiol. 1959 Oct;148:574-91. doi: 10.1113/jphysiol.1959.sp006308.},
   keywords = {Animals
Cats
Cerebral Cortex/*physiology
Neurons/*physiology
*Visual Cortex
*CEREBRAL CORTEX/physiology
*NEURONS/physiology},
   ISSN = {0022-3751 (Print)
0022-3751 (Linking)},
   DOI = {10.1113/jphysiol.1959.sp006308},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/14403679},
   year = {1959},
   type = {Journal Article}
}

@article{RN1237,
   author = {Hubel, D. H. and Wiesel, T. N.},
   title = {Receptive fields, binocular interaction and functional architecture in the cat's visual cortex},
   journal = {Journal of Physiology},
   volume = {160},
   pages = {106-154},
   note = {HUBEL, D H
WIESEL, T N
eng
England
J Physiol. 1962 Jan;160:106-54. doi: 10.1113/jphysiol.1962.sp006837.},
   keywords = {Animals
Cats
Cerebral Cortex/*physiology
*Visual Cortex
*CEREBRAL CORTEX/physiology},
   ISSN = {0022-3751 (Print)
0022-3751 (Linking)},
   DOI = {10.1113/jphysiol.1962.sp006837},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/14449617},
   year = {1962},
   type = {Journal Article}
}

@book{RN12255,
   author = {Ito, M.},
   title = {The Cerebellum and Neural Control},
   publisher = {Raven Press},
   url = {https://archive.org/details/cerebellumneural0000itom},
   year = {1984},
   type = {Book}
}

@article{RN3458,
   author = {Ito, M.},
   title = {Movement and thought: identical control mechanisms by the cerebellum},
   journal = {Trends Neurosci},
   volume = {16},
   number = {11},
   pages = {448-50; discussion 453-4},
   note = {Ito, M
eng
Comment
Review
England
1993/11/01
Trends Neurosci. 1993 Nov;16(11):448-50; discussion 453-4. doi: 10.1016/0166-2236(93)90073-u.},
   keywords = {Animals
Cerebellum/*physiology
Humans
Movement/*physiology
Thinking/*physiology},
   ISSN = {0166-2236 (Print)
0166-2236 (Linking)},
   DOI = {10.1016/0166-2236(93)90073-u},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/7507615},
   year = {1993},
   type = {Journal Article}
}

@article{RN11708,
   author = {Ito, M.},
   title = {Mechanisms of motor learning in the cerebellum},
   journal = {Brain Res},
   volume = {886},
   number = {1-2},
   pages = {237-245},
   note = {Ito, M
eng
Review
Netherlands
2000/12/20
Brain Res. 2000 Dec 15;886(1-2):237-245. doi: 10.1016/s0006-8993(00)03142-5.},
   abstract = {How the elaborate neuronal circuit in the cerebellum operates and is involved in motor learning is a question addressed in earnest in studies on the cerebellum. During the past four decades, experimental studies have revealed circuit and module structures of the cerebellum, established long-term depression (LTD) as a unique and characteristic type of synaptic plasticity in the cerebellum, and analysed signal contents of activates of cerebellar neurons related to motor learning. In the 1990s, these studies were developed to detailed analyses of the signal transduction underlying LTD, and to uncovering the involvement of the cerebellum in cognitive function. On the other hand, theoretical studies yielded epochal Marr-Albus network models of the cerebellum around 1970, and introduced control system principles explaining the essential roles of the cerebellum in motor learning as providing internal models, both forward and inverse. The author maintains the hypothesis that reorganisation of the neuronal circuit by error-driven induction of LTD constitutes the major memory and learning mechanisms of the cerebellum. In this article, we examine the validity of the hypothesis in light of currently available data in recent studies of the cerebellum.},
   keywords = {Animals
Cerebellum/*physiology
Feedback/physiology
Learning/*physiology
Models, Neurological
Motor Neurons/physiology
Motor Skills/*physiology
Nerve Net/*physiology
Neural Inhibition/physiology
Neuronal Plasticity/physiology
Signal Transduction/physiology},
   ISSN = {0006-8993 (Print)
0006-8993 (Linking)},
   DOI = {10.1016/s0006-8993(00)03142-5},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/11119699},
   year = {2000},
   type = {Journal Article}
}

@article{RN4491,
   author = {Ito, Makoto and Doya, Kenji},
   title = {Distinct neural representation in the dorsolateral, dorsomedial, and ventral parts of the striatum during fixed- and free-choice tasks},
   journal = {Journal of Neuroscience},
   volume = {35},
   number = {8},
   pages = {3499-3514},
   DOI = {10.1523/JNEUROSCI.1962-14.2015},
   year = {2015},
   type = {Journal Article}
}

@article{RN7416,
   author = {Jaderberg, M. and Czarnecki, W. M. and Dunning, I. and Marris, L. and Lever, G. and Castaneda, A. G. and Beattie, C. and Rabinowitz, N. C. and Morcos, A. S. and Ruderman, A. and Sonnerat, N. and Green, T. and Deason, L. and Leibo, J. Z. and Silver, D. and Hassabis, D. and Kavukcuoglu, K. and Graepel, T.},
   title = {Human-level performance in 3D multiplayer games with population-based reinforcement learning},
   journal = {Science},
   volume = {364},
   number = {6443},
   pages = {859-865},
   note = {Jaderberg, Max
Czarnecki, Wojciech M
Dunning, Iain
Marris, Luke
Lever, Guy
Castaneda, Antonio Garcia
Beattie, Charles
Rabinowitz, Neil C
Morcos, Ari S
Ruderman, Avraham
Sonnerat, Nicolas
Green, Tim
Deason, Louise
Leibo, Joel Z
Silver, David
Hassabis, Demis
Kavukcuoglu, Koray
Graepel, Thore
eng
Science. 2019 May 31;364(6443):859-865. doi: 10.1126/science.aau6249.},
   abstract = {Reinforcement learning (RL) has shown great success in increasingly complex single-agent environments and two-player turn-based games. However, the real world contains multiple agents, each learning and acting independently to cooperate and compete with other agents. We used a tournament-style evaluation to demonstrate that an agent can achieve human-level performance in a three-dimensional multiplayer first-person video game, Quake III Arena in Capture the Flag mode, using only pixels and game points scored as input. We used a two-tier optimization process in which a population of independent RL agents are trained concurrently from thousands of parallel matches on randomly generated environments. Each agent learns its own internal reward signal and rich representation of the world. These results indicate the great potential of multiagent reinforcement learning for artificial intelligence research.},
   ISSN = {1095-9203 (Electronic)
0036-8075 (Linking)},
   DOI = {10.1126/science.aau6249},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/31147514},
   year = {2019},
   type = {Journal Article}
}

@article{RN1949,
   author = {Kakade, S. and Dayan, P.},
   title = {Dopamine: generalization and bonuses},
   journal = {Neural Netw},
   volume = {15},
   number = {4-6},
   pages = {549-59},
   note = {Kakade, Sham
Dayan, Peter
eng
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Review
2002/10/10
Neural Netw. 2002 Jun-Jul;15(4-6):549-59. doi: 10.1016/s0893-6080(02)00048-5.},
   abstract = {In the temporal difference model of primate dopamine neurons, their phasic activity reports a prediction error for future reward. This model is supported by a wealth of experimental data. However, in certain circumstances, the activity of the dopamine cells seems anomalous under the model, as they respond in particular ways to stimuli that are not obviously related to predictions of reward. In this paper, we address two important sets of anomalies, those having to do with generalization and novelty. Generalization responses are treated as the natural consequence of partial information; novelty responses are treated by the suggestion that dopamine cells multiplex information about reward bonuses, including exploration bonuses and shaping bonuses. We interpret this additional role for dopamine in terms of the mechanistic attentional and psychomotor effects of dopamine, having the computational role of guiding exploration.},
   keywords = {Animals
Dopamine/*physiology
Exploratory Behavior/physiology
Generalization, Psychological/*physiology
Humans
*Reward},
   ISSN = {0893-6080 (Print)
0893-6080 (Linking)},
   DOI = {10.1016/s0893-6080(02)00048-5},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/12371511},
   year = {2002},
   type = {Journal Article}
}

@article{RN3693,
   author = {Kawato, M. and Furukawa, K. and Suzuki, R.},
   title = {A hierarchical neural-network model for control and learning of voluntary movement},
   journal = {Biol Cybern},
   volume = {57},
   number = {3},
   pages = {169-85},
   note = {Kawato, M
Furukawa, K
Suzuki, R
eng
Germany
1987/01/01
Biol Cybern. 1987;57(3):169-85. doi: 10.1007/BF00364149.},
   abstract = {In order to control voluntary movements, the central nervous system (CNS) must solve the following three computational problems at different levels: the determination of a desired trajectory in the visual coordinates, the transformation of its coordinates to the body coordinates and the generation of motor command. Based on physiological knowledge and previous models, we propose a hierarchical neural network model which accounts for the generation of motor command. In our model the association cortex provides the motor cortex with the desired trajectory in the body coordinates, where the motor command is then calculated by means of long-loop sensory feedback. Within the spinocerebellum--magnocellular red nucleus system, an internal neural model of the dynamics of the musculoskeletal system is acquired with practice, because of the heterosynaptic plasticity, while monitoring the motor command and the results of movement. Internal feedback control with this dynamical model updates the motor command by predicting a possible error of movement. Within the cerebrocerebellum--parvocellular red nucleus system, an internal neural model of the inverse-dynamics of the musculo-skeletal system is acquired while monitoring the desired trajectory and the motor command. The inverse-dynamics model substitutes for other brain regions in the complex computation of the motor command. The dynamics and the inverse-dynamics models are realized by a parallel distributed neural network, which comprises many sub-systems computing various nonlinear transformations of input signals and a neuron with heterosynaptic plasticity (that is, changes of synaptic weights are assumed proportional to a product of two kinds of synaptic inputs). Control and learning performance of the model was investigated by computer simulation, in which a robotic manipulator was used as a controlled system, with the following results: (1) Both the dynamics and the inverse-dynamics models were acquired during control of movements. (2) As motor learning proceeded, the inverse-dynamics model gradually took the place of external feedback as the main controller. Concomitantly, overall control performance became much better. (3) Once the neural network model learned to control some movement, it could control quite different and faster movements. (4) The neural network model worked well even when only very limited information about the fundamental dynamical structure of the controlled system was available.(ABSTRACT TRUNCATED AT 400 WORDS)},
   keywords = {Animals
Brain/*physiology
*Learning
Mathematics
*Models, Neurological
*Models, Psychological
*Motor Activity
Movement
Neuronal Plasticity
Neurons/*physiology
Stochastic Processes
Synapses/physiology},
   ISSN = {0340-1200 (Print)
0340-1200 (Linking)},
   DOI = {10.1007/BF00364149},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/3676355
https://link.springer.com/content/pdf/10.1007/BF00364149.pdf},
   year = {1987},
   type = {Journal Article}
}

@article{RN5030,
   author = {Kingma, Diederik P. and Welling, Max},
   title = {Auto-encoding variational Bayes},
   journal = {International Conference on Learning Representations (ICLR)},
   year = {2014},
   type = {Journal Article}
}

@article{RN1464,
   author = {Knill, D. C. and Pouget, A.},
   title = {The Bayesian brain: the role of uncertainty in neural coding and computation},
   journal = {Trends Neurosci},
   volume = {27},
   number = {12},
   pages = {712-9},
   note = {Knill, David C
Pouget, Alexandre
eng
England
Trends Neurosci. 2004 Dec;27(12):712-9.},
   abstract = {To use sensory information efficiently to make judgments and guide action in the world, the brain must represent and use information about uncertainty in its computations for perception and action. Bayesian methods have proven successful in building computational theories for perception and sensorimotor control, and psychophysics is providing a growing body of evidence that human perceptual computations are "Bayes' optimal". This leads to the "Bayesian coding hypothesis": that the brain represents sensory information probabilistically, in the form of probability distributions. Several computational schemes have recently been proposed for how this might be achieved in populations of neurons. Neurophysiological data on the hypothesis, however, is almost non-existent. A major challenge for neuroscientists is to test these ideas experimentally, and so determine whether and how neurons code information about sensory uncertainty.},
   keywords = {Animals
*Bayes Theorem
Brain/*physiology
Humans
Models, Biological
Nerve Net
Neurons/metabolism
*Perception},
   ISSN = {0166-2236 (Print)
0166-2236 (Linking)},
   DOI = {10.1016/j.tins.2004.10.007},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/15541511},
   year = {2004},
   type = {Journal Article}
}

@article{RN1484,
   author = {Kohonen, Teuvo},
   title = {Self-Organized Formation of Topologically Correct Feature Maps},
   journal = {Biological Cybernetics},
   volume = {43},
   pages = {59-69},
   DOI = {10.1007/BF00337288},
   year = {1982},
   type = {Journal Article}
}

@article{RN1480,
   author = {Kording, K. P. and Wolpert, D. M.},
   title = {Bayesian integration in sensorimotor learning},
   journal = {Nature},
   volume = {427},
   number = {6971},
   pages = {244-7},
   note = {Kording, Konrad P
Wolpert, Daniel M
eng
Research Support, Non-U.S. Gov't
England
2004/01/16
Nature. 2004 Jan 15;427(6971):244-7. doi: 10.1038/nature02169.},
   abstract = {When we learn a new motor skill, such as playing an approaching tennis ball, both our sensors and the task possess variability. Our sensors provide imperfect information about the ball's velocity, so we can only estimate it. Combining information from multiple modalities can reduce the error in this estimate. On a longer time scale, not all velocities are a priori equally probable, and over the course of a match there will be a probability distribution of velocities. According to bayesian theory, an optimal estimate results from combining information about the distribution of velocities-the prior-with evidence from sensory feedback. As uncertainty increases, when playing in fog or at dusk, the system should increasingly rely on prior knowledge. To use a bayesian strategy, the brain would need to represent the prior distribution and the level of uncertainty in the sensory feedback. Here we control the statistical variations of a new sensorimotor task and manipulate the uncertainty of the sensory feedback. We show that subjects internally represent both the statistical distribution of the task and their sensory uncertainty, combining them in a manner consistent with a performance-optimizing bayesian process. The central nervous system therefore employs probabilistic models during sensorimotor learning.},
   keywords = {*Bayes Theorem
Brain/*physiology
Feedback
Female
Fingers/physiology
Humans
Learning/*physiology
Male
Motor Skills/physiology
Movement
Normal Distribution
Photic Stimulation
Psychomotor Performance/*physiology},
   ISSN = {1476-4687 (Electronic)
0028-0836 (Linking)},
   DOI = {10.1038/nature02169},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/14724638},
   year = {2004},
   type = {Journal Article}
}

@article{RN6085,
   author = {Lake, B. M. and Ullman, T. D. and Tenenbaum, J. B. and Gershman, S. J.},
   title = {Building machines that learn and think like people},
   journal = {Behav Brain Sci},
   volume = {40},
   pages = {e253},
   note = {Lake, Brenden M
Ullman, Tomer D
Tenenbaum, Joshua B
Gershman, Samuel J
eng
England
Behav Brain Sci. 2017 Jan;40:e253. doi: 10.1017/S0140525X16001837. Epub 2016 Nov 24.},
   abstract = {Recent progress in artificial intelligence has renewed interest in building systems that learn and think like people. Many advances have come from using deep neural networks trained end-to-end in tasks such as object recognition, video games, and board games, achieving performance that equals or even beats that of humans in some respects. Despite their biological inspiration and performance achievements, these systems differ from human intelligence in crucial ways. We review progress in cognitive science suggesting that truly human-like learning and thinking machines will have to reach beyond current engineering trends in both what they learn and how they learn it. Specifically, we argue that these machines should (1) build causal models of the world that support explanation and understanding, rather than merely solving pattern recognition problems; (2) ground learning in intuitive theories of physics and psychology to support and enrich the knowledge that is learned; and (3) harness compositionality and learning-to-learn to rapidly acquire and generalize knowledge to new tasks and situations. We suggest concrete challenges and promising routes toward these goals that can combine the strengths of recent neural network advances with more structured cognitive models.},
   ISSN = {1469-1825 (Electronic)
0140-525X (Linking)},
   DOI = {10.1017/S0140525X16001837},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/27881212},
   year = {2017},
   type = {Journal Article}
}

@article{RN1615,
   author = {Linsker, Ralph},
   title = {From basic network principles to neural architecture: Emergence ofspatial-opponentcells},
   journal = {Proceedings of National Academy of Sciences, USA},
   volume = {83},
   number = {19},
   pages = {7508-7512},
   note = {Linsker, R
eng
Proc Natl Acad Sci U S A. 1986 Oct;83(19):7508-12. doi: 10.1073/pnas.83.19.7508.},
   abstract = {The functional architecture of mammalian visual cortex has been elucidated in impressive detail by experimental work of the past 20-25 years. The origin of many of the salient features of this architecture, however, has remained unexplained. This paper is the first of three (the others will appear in subsequent issues of these Proceedings) that address the origin and organization of feature-analyzing (spatial-opponent and orientation-selective) cells in simple systems governed by biologically plausible development rules. I analyze the progressive maturation of a system composed of a few layers of cells, with connections that develop according to a simple set of rules (including Hebb-type modification). To understand the prenatal origin of orientation-selective cells in certain primates, I consider the case in which there is no external input, with the first layer exhibiting random spontaneous electrical activity. No orientation preference is specified to the system at any stage, and none of the basic developmental rules is specific to visual processing. Here I introduce the theory of "modular self-adaptive networks," of which this system is an example, and explicitly demonstrate the emergence of a layer of spatial-opponent cells. This sets the stage for the emergence, in succeeding layers, of an orientation-selective cell population.},
   keywords = {Animals
Models, Theoretical
Nerve Net
Primates
Retina/*anatomy & histology/growth & development/physiology
Synapses/physiology
Visual Cortex/*anatomy & histology/growth & development/physiology
Visual Perception/*physiology},
   ISSN = {0027-8424 (Print)
0027-8424 (Linking)},
   DOI = {10.1073/pnas.83.19.7508},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/3463980},
   year = {1986},
   type = {Journal Article}
}

@article{RN1628,
   author = {Lochmann, T. and Deneve, S.},
   title = {Neural processing as causal inference},
   journal = {Curr Opin Neurobiol},
   volume = {21},
   number = {5},
   pages = {774-81},
   note = {Lochmann, Timm
Deneve, Sophie
eng
England
Curr Opin Neurobiol. 2011 Oct;21(5):774-81. doi: 10.1016/j.conb.2011.05.018.},
   abstract = {Perception is about making sense, that is, understanding what events in the outside world caused the sensory observations. Consistent with this intuition, many aspects of human behavior confronting noise and ambiguity are well explained by principles of causal inference. Extending these insights, recent studies have applied the same powerful set of tools to perceptual processing at the neural level. According to these approaches, microscopic neural structures solve elementary probabilistic tasks and can be combined to construct hierarchical predictive models of the sensory input. This framework suggests that variability in neural responses reflects the inherent uncertainty associated with sensory interpretations and that sensory neurons are active predictors rather than passive filters of their inputs. Causal inference can account parsimoniously and quantitatively for non-linear dynamical properties in single synapses, single neurons and sensory receptive fields.},
   keywords = {Action Potentials/physiology
Cerebral Cortex/cytology
Concept Formation/*physiology
Humans
*Models, Neurological
Nerve Net/physiology
Neural Networks (Computer)
Perception/*physiology
Probability
*Sensation
Sensory Receptor Cells/*physiology},
   ISSN = {1873-6882 (Electronic)
0959-4388 (Linking)},
   DOI = {10.1016/j.conb.2011.05.018},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/21742484},
   year = {2011},
   type = {Journal Article}
}

@article{RN6451,
   author = {Lottem, E. and Banerjee, D. and Vertechi, P. and Sarra, D. and Lohuis, M. O. and Mainen, Z. F.},
   title = {Activation of serotonin neurons promotes active persistence in a probabilistic foraging task},
   journal = {Nat Commun},
   volume = {9},
   number = {1},
   pages = {1000},
   note = {Lottem, Eran
Banerjee, Dhruba
Vertechi, Pietro
Sarra, Dario
Lohuis, Matthijs Oude
Mainen, Zachary F
eng
England
Nat Commun. 2018 Mar 8;9(1):1000. doi: 10.1038/s41467-018-03438-y.},
   abstract = {The neuromodulator serotonin (5-HT) has been implicated in a variety of functions that involve patience or impulse control. Many of these effects are consistent with a long-standing theory that 5-HT promotes behavioral inhibition, a motivational bias favoring passive over active behaviors. To further test this idea, we studied the impact of 5-HT in a probabilistic foraging task, in which mice must learn the statistics of the environment and infer when to leave a depleted foraging site for the next. Critically, mice were required to actively nose-poke in order to exploit a given site. We show that optogenetic activation of 5-HT neurons in the dorsal raphe nucleus increases the willingness of mice to actively attempt to exploit a reward site before giving up. These results indicate that behavioral inhibition is not an adequate description of 5-HT function and suggest that a unified account must be based on a higher-order function.},
   ISSN = {2041-1723 (Electronic)
2041-1723 (Linking)},
   DOI = {10.1038/s41467-018-03438-y},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/29520000},
   year = {2018},
   type = {Journal Article}
}

@article{RN1657,
   author = {Ma, W. J. and Beck, J. M. and Latham, P. E. and Pouget, A.},
   title = {Bayesian inference with probabilistic population codes},
   journal = {Nature Neuroscience},
   volume = {9},
   number = {11},
   pages = {1432-8},
   note = {Ma, Wei Ji
Beck, Jeffrey M
Latham, Peter E
Pouget, Alexandre
eng
5 T32 MH019942/MH/NIMH NIH HHS/
R01 MH62447/MH/NIMH NIH HHS/
T32 MH19942/MH/NIMH NIH HHS/
Nat Neurosci. 2006 Nov;9(11):1432-8. Epub 2006 Oct 22.},
   abstract = {Recent psychophysical experiments indicate that humans perform near-optimal Bayesian inference in a wide variety of tasks, ranging from cue integration to decision making to motor control. This implies that neurons both represent probability distributions and combine those distributions according to a close approximation to Bayes' rule. At first sight, it would seem that the high variability in the responses of cortical neurons would make it difficult to implement such optimal statistical inference in cortical circuits. We argue that, in fact, this variability implies that populations of neurons automatically represent probability distributions over the stimulus, a type of code we call probabilistic population codes. Moreover, we demonstrate that the Poisson-like variability observed in cortex reduces a broad class of Bayesian inference to simple linear combinations of populations of neural activity. These results hold for arbitrary probability distributions over the stimulus, for tuning curves of arbitrary shape and for realistic neuronal variability.},
   keywords = {Algorithms
*Bayes Theorem
Cerebral Cortex/*physiology
Humans
*Models, Neurological
*Models, Statistical
Nerve Net/cytology/*physiology
Normal Distribution
Poisson Distribution},
   ISSN = {1097-6256 (Print)
1097-6256 (Linking)},
   DOI = {10.1038/nn1790},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/17057707},
   year = {2006},
   type = {Journal Article}
}

@article{RN1688,
   author = {Marr, David},
   title = {A theory of cerebellar cortex},
   journal = {Journal of Physiology},
   volume = {202},
   pages = {437-370},
   DOI = {10.1113/jphysiol.1969.sp008820},
   year = {1969},
   type = {Journal Article}
}

@article{RN1689,
   author = {Marr, D.},
   title = {A theory for cerebral neocortex},
   journal = {Proc R Soc Lond B Biol Sci},
   volume = {176},
   number = {1043},
   pages = {161-234},
   note = {Marr, D
eng
England
1970/11/03
Proc R Soc Lond B Biol Sci. 1970 Nov 3;176(1043):161-234. doi: 10.1098/rspb.1970.0040.},
   keywords = {*Cerebral Cortex
Concept Formation
*Information Theory
*Models, Neurological
Neurophysiology},
   ISSN = {0950-1193 (Print)
0950-1193 (Linking)},
   DOI = {10.1098/rspb.1970.0040},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/4394740},
   year = {1970},
   type = {Journal Article}
}

@book{RN9659,
   author = {Marr, David},
   title = {Vision: A Computational Investigation into the Human Representation and Processing of Visual Information},
   publisher = {MIT Press},
   year = {1982},
   type = {Book}
}

@article{RN12254,
   author = {McCormick, D. A. and Thompson, R. F.},
   title = {Neuronal responses of the rabbit cerebellum during acquisition and performance of a classically conditioned nictitating membrane-eyelid response},
   journal = {J Neurosci},
   volume = {4},
   number = {11},
   pages = {2811-22},
   note = {McCormick, D A
Thompson, R F
eng
1-F31-MH08673/MH/NIMH NIH HHS/
Research Support, U.S. Gov't, Non-P.H.S.
Research Support, U.S. Gov't, P.H.S.
1984/11/01
J Neurosci. 1984 Nov;4(11):2811-22. doi: 10.1523/JNEUROSCI.04-11-02811.1984.},
   abstract = {Neuronal activity was recorded from regions of the cerebellar cortex and dentate-interpositus nuclei during learning and/or performance of a classically conditioned nictitating membrane (NM-a third eyelid)/eyeblink response in the rabbit. It was found that neurons located within restricted portions of the ansiform lobule and anterior lobe cortical regions and of the dentate-interpositus nuclei respond in relation to the performance of the learned eyeblink response. Furthermore, chronic recordings from the dentate-interpositus nuclei revealed that these responses develop in close relation to the learning of the conditioned eyeblink response. Stimulation of the dentate-interpositus nuclei through the recording electrodes in some cases yielded eyelid closure and NM extension in both trained and untrained animals. Lesion of the axons of the dentate-interpositus nuclei (superior cerebellar peduncle), a manipulation which is known to abolish the learned eyeblink response, abolished the stimulation effect. We have previously reported that lesions of the dentate-interpositus nuclei cause abolition of the learned eyeblink response. In the present study, we report that lesions of the regions of cerebellar cortex projecting to the dentate-interpositus nuclei do not permanently abolish the conditioned response, although the amplitude-time course of the learned response could be affected. These results, together with results of other studies, demonstrate that the medial dentate and/or lateral interpositus nuclei are active during learning and performance of the conditioned eyeblink response, are capable of producing this learned response, and are essential for the learning and retention of the conditioned eyeblink response. Therefore, the medial dentate and/or lateral interpositus nuclei are a part of the essential neuronal circuit involved in the learning and production of the classically conditioned eyeblink response in the rabbit.},
   keywords = {Acoustic Stimulation
Action Potentials
Animals
Cerebellar Cortex/physiology
Cerebellar Nuclei/physiology
Cerebellum/*physiology
Conditioning, Eyelid/*physiology
Electric Stimulation
Male
Neurons/*physiology
Nictitating Membrane/physiology
Rabbits},
   ISSN = {0270-6474 (Print)
1529-2401 (Electronic)
0270-6474 (Linking)},
   DOI = {10.1523/JNEUROSCI.04-11-02811.1984},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/6502205
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6564737/pdf/jneuro_4_11_2811.pdf},
   year = {1984},
   type = {Journal Article}
}

@book{RN3870,
   author = {Minsky, M},
   title = {Society of Mind},
   publisher = {Simon and Schuster},
   address = {New York},
   year = {1986},
   type = {Book}
}

@article{RN804,
   author = {Miyazaki, K. and Miyazaki, K. W. and Doya, K.},
   title = {Activation of dorsal raphe serotonin neurons underlies waiting for delayed rewards},
   journal = {J Neurosci},
   volume = {31},
   number = {2},
   pages = {469-79},
   note = {Miyazaki, Katsuhiko
Miyazaki, Kayoko W
Doya, Kenji
eng
J Neurosci. 2011 Jan 12;31(2):469-79. doi: 10.1523/JNEUROSCI.3714-10.2011.},
   abstract = {The serotonergic system plays a key role in the control of impulsive behaviors. Forebrain serotonin depletion leads to premature actions and steepens discounting of delayed rewards. However, there has been no direct evidence for serotonin neuron activity in relation to actions for delayed rewards. Here we show that serotonin neurons increase their tonic firing while rats wait for food and water rewards and conditioned reinforcement tones. The rate of tonic firing during the delay period was significantly higher for rewards than for tones, for which rats could not wait as long. When the delay was extended, tonic firing persisted until reward or tone delivery. When rats gave up waiting because of extended delay or reward omission, serotonin neuron firing dropped preceding the exit from reward sites. Serotonin neurons did not show significant response when an expected reward was omitted, which was predicted by the theory that serotonin signals negative reward prediction errors. These results suggest that increased serotonin neuron firing facilitates a rat's waiting behavior in prospect of forthcoming rewards and that higher serotonin activation enables longer waiting.},
   keywords = {Action Potentials
Animals
Conditioning, Operant
Male
Neurons/*physiology
Raphe Nuclei/*physiology
Rats
Rats, Long-Evans
Reaction Time
Reinforcement (Psychology)
*Reward
Serotonin/*metabolism
Time Factors},
   ISSN = {1529-2401 (Electronic)
0270-6474 (Linking)},
   DOI = {10.1523/JNEUROSCI.3714-10.2011},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/21228157},
   year = {2011},
   type = {Journal Article}
}

@article{RN6625,
   author = {Miyazaki, K. and Miyazaki, K. W. and Yamanaka, A. and Tokuda, T. and Tanaka, K. F. and Doya, K.},
   title = {Reward probability and timing uncertainty alter the effect of dorsal raphe serotonin neurons on patience},
   journal = {Nat Commun},
   volume = {9},
   number = {1},
   pages = {2048},
   note = {Miyazaki, Katsuhiko
Miyazaki, Kayoko W
Yamanaka, Akihiro
Tokuda, Tomoki
Tanaka, Kenji F
Doya, Kenji
eng
England
Nat Commun. 2018 Jun 1;9(1):2048. doi: 10.1038/s41467-018-04496-y.},
   abstract = {Recent experiments have shown that optogenetic activation of serotonin neurons in the dorsal raphe nucleus (DRN) in mice enhances patience in waiting for future rewards. Here, we show that serotonin effect in promoting waiting is maximized by both high probability and high timing uncertainty of reward. Optogenetic activation of serotonergic neurons prolongs waiting time in no-reward trials in a task with 75% food reward probability, but not with 50 or 25% reward probabilities. Serotonin effect in promoting waiting increases when the timing of reward presentation becomes unpredictable. To coherently explain the experimental data, we propose a Bayesian decision model of waiting that assumes that serotonin neuron activation increases the prior probability or subjective confidence of reward delivery. The present data and modeling point to the possibility of a generalized role of serotonin in resolving trade-offs, not only between immediate and delayed rewards, but also between sensory evidence and subjective confidence.},
   ISSN = {2041-1723 (Electronic)
2041-1723 (Linking)},
   DOI = {10.1038/s41467-018-04496-y},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/29858574},
   year = {2018},
   type = {Journal Article}
}

@article{RN4230,
   author = {Miyazaki, K. W. and Miyazaki, K. and Tanaka, K. F. and Yamanaka, A. and Takahashi, A. and Tabuchi, S. and Doya, K.},
   title = {Optogenetic activation of dorsal raphe serotonin neurons enhances patience for future rewards},
   journal = {Current Biology},
   volume = {24},
   number = {17},
   pages = {2033-40},
   note = {Miyazaki, Kayoko W
Miyazaki, Katsuhiko
Tanaka, Kenji F
Yamanaka, Akihiro
Takahashi, Aki
Tabuchi, Sawako
Doya, Kenji
eng
England
2014/08/27 06:00
Curr Biol. 2014 Sep 8;24(17):2033-40. doi: 10.1016/j.cub.2014.07.041. Epub 2014 Aug 21.},
   abstract = {Serotonin is a neuromodulator that is involved extensively in behavioral, affective, and cognitive functions in the brain. Previous recording studies of the midbrain dorsal raphe nucleus (DRN) revealed that the activation of putative serotonin neurons correlates with the levels of behavioral arousal [1], rhythmic motor outputs [2], salient sensory stimuli [3-6], reward, and conditioned cues [5-8]. The classic theory on serotonin states that it opposes dopamine and inhibits behaviors when aversive events are predicted [9-14]. However, the therapeutic effects of serotonin signal-enhancing medications have been difficult to reconcile with this theory [15, 16]. In contrast, a more recent theory states that serotonin facilitates long-term optimal behaviors and suppresses impulsive behaviors [17-21]. To test these theories, we developed optogenetic mice that selectively express channelrhodopsin in serotonin neurons and tested how the activation of serotonergic neurons in the DRN affects animal behavior during a delayed reward task. The activation of serotonin neurons reduced the premature cessation of waiting for conditioned cues and food rewards. In reward omission trials, serotonin neuron stimulation prolonged the time animals spent waiting. This effect was observed specifically when the animal was engaged in deciding whether to keep waiting and was not due to motor inhibition. Control experiments showed that the prolonged waiting times observed with optogenetic stimulation were not due to behavioral inhibition or the reinforcing effects of serotonergic activation. These results show, for the first time, that the timed activation of serotonin neurons during waiting promotes animals' patience to wait for a delayed reward.},
   ISSN = {1879-0445 (Electronic)
0960-9822 (Linking)},
   DOI = {10.1016/j.cub.2014.07.041},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/25155504},
   year = {2014},
   type = {Journal Article}
}

@book{RN5901,
   author = {Murphy, Kevin P.},
   title = {Machine Learning: A Probabilistic Perspective},
   publisher = {MIT Press},
   ISBN = {9780262018029},
   url = {https://probml.github.io/pml-book/},
   year = {2012},
   type = {Book}
}

@article{RN8708,
   author = {Niv, Yael and Joel, Daphna and Meilijson, Isaac and Ruppin, Eytan},
   title = {Evolution of Reinforcement Learning in Uncertain Environments: A Simple Explanation for Complex Foraging Behaviors},
   journal = {Adaptive Behavior},
   volume = {10},
   number = {1},
   pages = {5-24},
   ISSN = {1059-7123
1741-2633},
   DOI = {10.1177/1059-712302-010001-01},
   year = {2002},
   type = {Journal Article}
}

@article{RN1968,
   author = {Nomoto, K. and Schultz, W. and Watanabe, T. and Sakagami, M.},
   title = {Temporally extended dopamine responses to perceptually demanding reward-predictive stimuli},
   journal = {J Neurosci},
   volume = {30},
   number = {32},
   pages = {10692-702},
   note = {Nomoto, Kensaku
Schultz, Wolfram
Watanabe, Takeo
Sakagami, Masamichi
eng
095495/Wellcome Trust/United Kingdom
R01 EY015980/EY/NEI NIH HHS/
R01 EY015980-04A2/EY/NEI NIH HHS/
R01 EY015980-05/EY/NEI NIH HHS/
R01 EY015980-06/EY/NEI NIH HHS/
R01 EY019466/EY/NEI NIH HHS/
R01 EY019466-01/EY/NEI NIH HHS/
R01 EY019466-02/EY/NEI NIH HHS/
R01 EY019466-03/EY/NEI NIH HHS/
R01 EY019466-04/EY/NEI NIH HHS/
Wellcome Trust/United Kingdom
J Neurosci. 2010 Aug 11;30(32):10692-702. doi: 10.1523/JNEUROSCI.4828-09.2010.},
   abstract = {Midbrain dopamine neurons respond to reward-predictive stimuli. In the natural environment reward-predictive stimuli are often perceptually complicated. Thus, to discriminate one stimulus from another, elaborate sensory processing is necessary. Given that previous studies have used simpler types of reward-predictive stimuli, it has yet to be clear whether and, if so, how dopamine neurons obtain reward information from perceptually complicated stimuli. To investigate this, we recorded the activities of monkey dopamine neurons while they were performing discrimination between two coherent motion directions in random-dot motion stimuli. These coherent directions were paired with different magnitudes of reward. We found that dopamine neurons showed reward-predictive responses to random-dot motion stimuli. Moreover, dopamine neurons showed temporally extended activity correlated with changes in reward prediction (i.e., reward prediction error) from coarse to fine scales between initial motion detection and subsequent motion discrimination phases. Noticeably, dopamine reward-predictive responses became differential in a later phase than previously reported. This response pattern was consistent with the time course of processing required for the estimation of expected reward value that parallels the motion direction discrimination processing. The results demonstrate that dopamine neurons are able to reflect the reward value of perceptually complicated stimuli, and suggest that dopamine neurons use the moment-to-moment reward prediction associated with environmental stimuli to compute a reward prediction error.},
   keywords = {Acoustic Stimulation/methods
Analysis of Variance
Animals
Behavior, Animal
Conditioning, Operant/physiology
Discrimination (Psychology)/*physiology
Dopamine/*metabolism
Feedback, Physiological/physiology
Functional Laterality/physiology
Macaca fascicularis
Magnetic Resonance Imaging/methods
Male
Mesencephalon/cytology
Motion Perception/*physiology
Neurons/*physiology
Orientation/physiology
Photic Stimulation/methods
Predictive Value of Tests
*Reward
Saccades/physiology
Time Factors
Tyrosine 3-Monooxygenase/metabolism},
   ISSN = {1529-2401 (Electronic)
0270-6474 (Linking)},
   DOI = {10.1523/JNEUROSCI.4828-09.2010},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/20702700},
   year = {2010},
   type = {Journal Article}
}

@article{RN6630,
   author = {Ogasawara, H. and Doi, T. and Kawato, M.},
   title = {Systems biology perspectives on cerebellar long-term depression},
   journal = {Neurosignals},
   volume = {16},
   number = {4},
   pages = {300-17},
   note = {Ogasawara, Hideaki
Doi, Tomokazu
Kawato, Mitsuo
eng
Research Support, Non-U.S. Gov't
Review
Switzerland
Neurosignals. 2008;16(4):300-17. doi: 10.1159/000123040. Epub 2008 Jul 18.},
   abstract = {Long-term depression (LTD) at parallel fiber-Purkinje cell (PF-PC) synapses is thought to be the cellular correlate of cerebellar associative learning. The molecular processes are, in brief, phosphorylation of AMPA-type glutamate receptors (AMPARs) and their subsequent removal from the surface of the PF-PC synapse. In order to elucidate the fundamental mechanisms for cerebellar LTD and further the understanding of its computational role, we have investigated its systems biology and proposed the following hypotheses, some of which have already been experimentally verified: (1) due to the mitogen-activated protein kinase (MAPK)-protein kinase C (PKC) positive feedback loop, phosphorylation of AMPARs is an all-or-none event; (2) the inositol 1,4,5-triphosphate receptor detects concurrent PF and climbing fiber inputs, forming the cellular basis for associative learning, and (3) the local concentration of nitric oxide in the PC dendrite reflects the relevance of a given context, enabling context-dependent selection of learning modules within the cerebellum. In this review, we first introduce theoretical studies on cerebellar LTD, mainly focusing on our own published work, followed by a discussion of the effects of stochasticity, localization, diffusion, and scaffolding. Neurons embody two features that are apparently contradictory, yet necessary for synaptic memory: stability and plasticity. We will also present models for explaining how neurons solve this dilemma. In the final section, we propose a conceptual model in which a cascade of excitable dynamics with different time scales, i.e., Ca(2+)-induced Ca(2+) release, the MAPK-PKC positive feedback loop, and protein kinase Mzeta (PKMzeta)-induced PKMzeta synthesis, provides a mechanism for stable memory that is still amenable to modifications.},
   keywords = {Animals
Association Learning/*physiology
Calcium Signaling
Cerebellum/*physiology
Computer Simulation
Feedback, Physiological
Humans
Long-Term Synaptic Depression/*physiology
MAP Kinase Signaling System
*Models, Neurological
Nerve Tissue Proteins/physiology
Nitric Oxide/physiology
Phosphorylation
Protein Kinase C/physiology
Protein Processing, Post-Translational
Purkinje Fibers/physiology
Rats
Receptors, AMPA/physiology
Synapses/physiology
*Systems Biology},
   ISSN = {1424-8638 (Electronic)
1424-862X (Linking)},
   DOI = {10.1159/000123040},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/18635946},
   year = {2008},
   type = {Journal Article}
}

@article{RN2011,
   author = {Olshausen, B. A. and Field, D. J.},
   title = {Emergence of simple-cell receptive field properties by learning a sparse code for natural images},
   journal = {Nature},
   volume = {381},
   number = {6583},
   pages = {607-9},
   note = {Olshausen, B A
Field, D J
eng
Research Support, U.S. Gov't, P.H.S.
England
1996/06/13
Nature. 1996 Jun 13;381(6583):607-9. doi: 10.1038/381607a0.},
   abstract = {The receptive fields of simple cells in mammalian primary visual cortex can be characterized as being spatially localized, oriented and bandpass (selective to structure at different spatial scales), comparable to the basis functions of wavelet transforms. One approach to understanding such response properties of visual neurons has been to consider their relationship to the statistical structure of natural images in terms of efficient coding. Along these lines, a number of studies have attempted to train unsupervised learning algorithms on natural images in the hope of developing receptive fields with similar properties, but none has succeeded in producing a full set that spans the image space and contains all three of the above properties. Here we investigate the proposal that a coding strategy that maximizes sparseness is sufficient to account for these properties. We show that a learning algorithm that attempts to find sparse linear codes for natural scenes will develop a complete family of localized, oriented, bandpass receptive fields, similar to those found in the primary visual cortex. The resulting sparse image code provides a more efficient representation for later stages of processing because it possesses a higher degree of statistical independence among its outputs.},
   keywords = {Algorithms
Learning
Models, Neurological
Neurons/*physiology
Vision, Ocular/*physiology
Visual Cortex/cytology/*physiology},
   ISSN = {0028-0836 (Print)
0028-0836 (Linking)},
   DOI = {10.1038/381607a0},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/8637596},
   year = {1996},
   type = {Journal Article}
}

@article{RN3907,
   author = {Rao, R. P. and Ballard, D. H.},
   title = {Predictive coding in the visual cortex: a functional interpretation of some extra-classical receptive-field effects},
   journal = {Nat Neurosci},
   volume = {2},
   number = {1},
   pages = {79-87},
   note = {Rao, R P
Ballard, D H
eng
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Research Support, U.S. Gov't, P.H.S.
1999/04/09
Nat Neurosci. 1999 Jan;2(1):79-87. doi: 10.1038/4580.},
   abstract = {We describe a model of visual processing in which feedback connections from a higher- to a lower-order visual cortical area carry predictions of lower-level neural activities, whereas the feedforward connections carry the residual errors between the predictions and the actual lower-level activities. When exposed to natural images, a hierarchical network of model neurons implementing such a model developed simple-cell-like receptive fields. A subset of neurons responsible for carrying the residual errors showed endstopping and other extra-classical receptive-field effects. These results suggest that rather than being exclusively feedforward phenomena, nonclassical surround effects in the visual cortex may also result from cortico-cortical feedback as a consequence of the visual system using an efficient hierarchical strategy for encoding natural images.},
   keywords = {Feedback
Forecasting
*Models, Neurological
Neural Networks, Computer
Visual Cortex/*physiology
Visual Pathways/physiology},
   ISSN = {1097-6256 (Print)
1097-6256 (Linking)},
   DOI = {10.1038/4580},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/10195184},
   year = {1999},
   type = {Journal Article}
}

@misc{RN2180,
   author = {Reisinger, Joseph and Miikkulainen, Risto},
   title = {Acquiring Evolvability through Adaptive Representations},
   year = {2007},
   type = {Conference Paper}
}

@article{RN6707,
   author = {Rilling, James K. and Gutman, David A. and Zeh, Thorsten R. and Pagnoni, Giuseppe and Berns, Gregory S. and Kilts, Clinton D.},
   title = {A Neural Basis for Social Cooperation},
   journal = {Neuron},
   volume = {35},
   number = {2},
   pages = {395-405},
   ISSN = {08966273},
   DOI = {10.1016/s0896-6273(02)00755-9},
   year = {2002},
   type = {Journal Article}
}

@article{RN3717,
   author = {Rosenblatt, F.},
   title = {Principles of Neurodynamics},
   pages = {Spartan},
   year = {1962},
   type = {Journal Article}
}

@article{RN2268,
   author = {Salakhutdinov, R. and Hinton, G.},
   title = {An efficient learning procedure for deep Boltzmann machines},
   journal = {Neural Comput},
   volume = {24},
   number = {8},
   pages = {1967-2006},
   note = {Salakhutdinov, Ruslan
Hinton, Geoffrey
eng
Research Support, Non-U.S. Gov't
2012/04/19
Neural Comput. 2012 Aug;24(8):1967-2006. doi: 10.1162/NECO_a_00311. Epub 2012 Apr 17.},
   abstract = {We present a new learning algorithm for Boltzmann machines that contain many layers of hidden variables. Data-dependent statistics are estimated using a variational approximation that tends to focus on a single mode, and data-independent statistics are estimated using persistent Markov chains. The use of two quite different techniques for estimating the two types of statistic that enter into the gradient of the log likelihood makes it practical to learn Boltzmann machines with multiple hidden layers and millions of parameters. The learning can be made more efficient by using a layer-by-layer pretraining phase that initializes the weights sensibly. The pretraining also allows the variational inference to be initialized sensibly with a single bottom-up pass. We present results on the MNIST and NORB data sets showing that deep Boltzmann machines learn very good generative models of handwritten digits and 3D objects. We also show that the features discovered by deep Boltzmann machines are a very effective way to initialize the hidden layers of feedforward neural nets, which are then discriminatively fine-tuned.},
   ISSN = {1530-888X (Electronic)
0899-7667 (Linking)},
   DOI = {10.1162/NECO_a_00311},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/22509963},
   year = {2012},
   type = {Journal Article}
}

@article{RN840,
   author = {Samejima, K. and Ueda, Y. and Doya, K. and Kimura, M.},
   title = {Representation of action-specific reward values in the striatum},
   journal = {Science},
   volume = {310},
   number = {5752},
   pages = {1337-40},
   note = {Samejima, Kazuyuki
Ueda, Yasumasa
Doya, Kenji
Kimura, Minoru
eng
New York, N.Y.
Science. 2005 Nov 25;310(5752):1337-40.},
   abstract = {The estimation of the reward an action will yield is critical in decision-making. To elucidate the role of the basal ganglia in this process, we recorded striatal neurons of monkeys who chose between left and right handle turns, based on the estimated reward probabilities of the actions. During a delay period before the choices, the activity of more than one-third of striatal projection neurons was selective to the values of one of the two actions. Fewer neurons were tuned to relative values or action choice. These results suggest representation of action values in the striatum, which can guide action selection in the basal ganglia circuit.},
   keywords = {Action Potentials
Animals
Brain Mapping
Caudate Nucleus/*physiology
*Choice Behavior
Corpus Striatum/*physiology
Female
Macaca
Male
Neurons/*physiology
Probability
Putamen/*physiology
Regression Analysis
Reinforcement (Psychology)
*Reward},
   ISSN = {1095-9203 (Electronic)
0036-8075 (Linking)},
   DOI = {10.1126/science.1115270},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/16311337},
   year = {2005},
   type = {Journal Article}
}

@article{RN2277,
   author = {Sanfey, A. G. and Rilling, J. K. and Aronson, J. A. and Nystrom, L. E. and Cohen, J. D.},
   title = {The neural basis of economic decision-making in the Ultimatum Game},
   journal = {Science},
   volume = {300},
   number = {5626},
   pages = {1755-8},
   note = {Sanfey, Alan G
Rilling, James K
Aronson, Jessica A
Nystrom, Leigh E
Cohen, Jonathan D
eng
New York, N.Y.
Science. 2003 Jun 13;300(5626):1755-8.},
   abstract = {The nascent field of neuroeconomics seeks to ground economic decision making in the biological substrate of the brain. We used functional magnetic resonance imaging of Ultimatum Game players to investigate neural substrates of cognitive and emotional processes involved in economic decision-making. In this game, two players split a sum of money;one player proposes a division and the other can accept or reject this. We scanned players as they responded to fair and unfair proposals. Unfair offers elicited activity in brain areas related to both emotion (anterior insula) and cognition (dorsolateral prefrontal cortex). Further, significantly heightened activity in anterior insula for rejected unfair offers suggests an important role for emotions in decision-making.},
   keywords = {Adult
Behavior
Brain/*physiology
Cerebral Cortex/physiology
Cognition
*Decision Making
*Economics
*Emotions
Female
Game Theory
*Games, Experimental
Gyrus Cinguli/physiology
Humans
Interpersonal Relations
Magnetic Resonance Imaging
Male
Prefrontal Cortex/physiology},
   ISSN = {1095-9203 (Electronic)
0036-8075 (Linking)},
   DOI = {10.1126/science.1082976},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/12805551},
   year = {2003},
   type = {Journal Article}
}

@article{RN2282,
   author = {Sanger, Terence D.},
   title = {Optimal unsupervised learning in a single-layer linear feedforward neural network},
   journal = {Neural Networks},
   volume = {2},
   number = {6},
   pages = {459-473},
   ISSN = {08936080},
   DOI = {10.1016/0893-6080(89)90044-0},
   year = {1989},
   type = {Journal Article}
}

@article{RN2291,
   author = {Sato, Y. and Akiyama, E. and Farmer, J. D.},
   title = {Chaos in learning a simple two-person game},
   journal = {Proc Natl Acad Sci U S A},
   volume = {99},
   number = {7},
   pages = {4748-51},
   note = {Sato, Yuzuru
Akiyama, Eizo
Farmer, J Doyne
eng
Proc Natl Acad Sci U S A. 2002 Apr 2;99(7):4748-51.},
   abstract = {We investigate the problem of learning to play the game of rock-paper-scissors. Each player attempts to improve her/his average score by adjusting the frequency of the three possible responses, using reinforcement learning. For the zero sum game the learning process displays Hamiltonian chaos. Thus, the learning trajectory can be simple or complex, depending on initial conditions. We also investigate the non-zero sum case and show that it can give rise to chaotic transients. This is, to our knowledge, the first demonstration of Hamiltonian chaos in learning a basic two-person game, extending earlier findings of chaotic attractors in dissipative systems. As we argue here, chaos provides an important self-consistency condition for determining when players will learn to behave as though they were fully rational. That chaos can occur in learning a simple game indicates one should use caution in assuming real people will learn to play a game according to a Nash equilibrium strategy.},
   keywords = {Game Theory
*Games, Experimental
Humans
*Learning},
   ISSN = {0027-8424 (Print)
0027-8424 (Linking)},
   DOI = {10.1073/pnas.032086299},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/11930020},
   year = {2002},
   type = {Journal Article}
}

@article{RN2334,
   author = {Schultz, Wolfram and Dayan, Peter and Montague, P. Read},
   title = {A neural substrate of prediction and reward},
   journal = {Science},
   volume = {275},
   pages = {1593-1599},
   DOI = {10.1126/science.275.5306.1593},
   year = {1997},
   type = {Journal Article}
}

@article{RN3202,
   author = {Schweighofer, N. and Doya, K.},
   title = {Meta-learning in reinforcement learning},
   journal = {Neural Netw},
   volume = {16},
   number = {1},
   pages = {5-9},
   note = {Schweighofer, Nicolas
Doya, Kenji
eng
Research Support, Non-U.S. Gov't
2003/02/11
Neural Netw. 2003 Jan;16(1):5-9. doi: 10.1016/s0893-6080(02)00228-9.},
   abstract = {Meta-parameters in reinforcement learning should be tuned to the environmental dynamics and the animal performance. Here, we propose a biologically plausible meta-reinforcement learning algorithm for tuning these meta-parameters in a dynamic, adaptive manner. We tested our algorithm in both a simulation of a Markov decision task and in a non-linear control task. Our results show that the algorithm robustly finds appropriate meta-parameter values, and controls the meta-parameter time course, in both static and dynamic environments. We suggest that the phasic and tonic components of dopamine neuron firing can encode the signal required for meta-learning of reinforcement learning.},
   keywords = {Algorithms
*Artificial Intelligence
Decision Theory
Markov Chains
*Reinforcement, Psychology
Stochastic Processes},
   ISSN = {0893-6080 (Print)
0893-6080 (Linking)},
   DOI = {10.1016/s0893-6080(02)00228-9},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/12576101},
   year = {2003},
   type = {Journal Article}
}

@book{RN7346,
   author = {Sutton, R. S. and Barto, A. G.},
   title = {Reinforcement Learning: An Introduction},
   publisher = {MIT Press},
   address = {Cambridge, MA},
   edition = {2nd Edition},
   url = {http://incompleteideas.net/book/the-book.html},
   year = {2018},
   type = {Book}
}

@article{RN2520,
   author = {Suzuki, S. and Harasawa, N. and Ueno, K. and Gardner, J. L. and Ichinohe, N. and Haruno, M. and Cheng, K. and Nakahara, H.},
   title = {Learning to simulate others' decisions},
   journal = {Neuron},
   volume = {74},
   number = {6},
   pages = {1125-37},
   note = {Suzuki, Shinsuke
Harasawa, Norihiro
Ueno, Kenichi
Gardner, Justin L
Ichinohe, Noritaka
Haruno, Masahiko
Cheng, Kang
Nakahara, Hiroyuki
eng
Neuron. 2012 Jun 21;74(6):1125-37. doi: 10.1016/j.neuron.2012.04.030.},
   abstract = {A fundamental challenge in social cognition is how humans learn another person's values to predict their decision-making behavior. This form of learning is often assumed to require simulation of the other by direct recruitment of one's own valuation process to model the other's process. However, the cognitive and neural mechanism of simulation learning is not known. Using behavior, modeling, and fMRI, we show that simulation involves two learning signals in a hierarchical arrangement. A simulated-other's reward prediction error processed in ventromedial prefrontal cortex mediated simulation by direct recruitment, being identical for valuation of the self and simulated-other. However, direct recruitment was insufficient for learning, and also required observation of the other's choices to generate a simulated-other's action prediction error encoded in dorsomedial/dorsolateral prefrontal cortex. These findings show that simulation uses a core prefrontal circuit for modeling the other's valuation to generate prediction and an adjunct circuit for tracking behavioral variation to refine prediction.},
   keywords = {Decision Making/*physiology
Humans
Image Processing, Computer-Assisted
*Interpersonal Relations
Learning/physiology
Magnetic Resonance Imaging
Models, Psychological
Prefrontal Cortex/*physiology
Social Behavior
*Social Perception},
   ISSN = {1097-4199 (Electronic)
0896-6273 (Linking)},
   DOI = {10.1016/j.neuron.2012.04.030},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/22726841},
   year = {2012},
   type = {Journal Article}
}

@article{RN5654,
   author = {Taylor, M. E. and Stone, P.},
   title = {Transfer Learning for Reinforcement Learning Domains: A Survey},
   journal = {Journal of Machine Learning Research},
   volume = {10},
   pages = {1633-1685},
   note = {507bm
Times Cited:769
Cited References Count:129},
   abstract = {The reinforcement learning paradigm is a popular way to address problems that have only limited environmental feedback, rather than correctly labeled examples, as is common in other machine learning contexts. While significant progress has been made to improve learning in a single task, the idea of transfer learning has only recently been applied to reinforcement learning tasks. The core idea of transfer is that experience gained in learning to perform one task can help improve learning performance in a related, but different, task. In this article we present a framework that classifies transfer learning methods in terms of their capabilities and goals, and then use it to survey the existing literature, as well as to suggest future directions for transfer learning work.},
   keywords = {transfer learning
reinforcement learning
multi-task learning
composing solutions
framework},
   ISSN = {1532-4435},
   url = {https://www.jmlr.org/papers/v10/taylor09a.html},
   year = {2009},
   type = {Journal Article}
}

@article{RN7439,
   author = {Thompson, R. F.},
   title = {The neurobiology of learning and memory},
   journal = {Science},
   volume = {233},
   number = {4767},
   pages = {941-7},
   note = {Thompson, R F
eng
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Science. 1986 Aug 29;233(4767):941-7. doi: 10.1126/science.3738519.},
   abstract = {Study of the neurobiology of learning and memory is in a most exciting phase. Behavioral studies in animals are characterizing the categories and properties of learning and memory; essential memory trace circuits in the brain are being defined and localized in mammalian models; work on human memory and the brain is identifying neuronal systems involved in memory; the neuronal, neurochemical, molecular, and biophysical substrates of memory are beginning to be understood in both invertebrate and vertebrate systems; and theoretical and mathematical analysis of basic associative learning and of neuronal networks in proceeding apace. Likely applications of this new understanding of the neural bases of learning and memory range from education to the treatment of learning disabilities to the design of new artificial intelligence systems.},
   keywords = {Animals
Aplysia/physiology
Brain/physiology
Cerebellum/physiology
Cerebral Cortex/physiology
Conditioning, Classical/physiology
Hippocampus/physiology
Humans
Learning/*physiology
Memory/*physiology
Neural Pathways/physiology
Neurons/physiology
Primates
Rats
Synapses/physiology},
   ISSN = {0036-8075 (Print)
0036-8075 (Linking)},
   DOI = {10.1126/science.3738519},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/3738519
https://science.sciencemag.org/content/sci/233/4767/941.full.pdf},
   year = {1986},
   type = {Journal Article}
}

@book{RN6564,
   author = {Thrun, Sebastian and Pratt, Lorien},
   title = {Learning to Learn},
   publisher = {Springer},
   DOI = {10.1007/978-1-4615-5529-2},
   year = {1998},
   type = {Edited Book}
}

@article{RN2616,
   author = {Todorov, E.},
   title = {Efficient computation of optimal actions},
   journal = {Proc Natl Acad Sci U S A},
   volume = {106},
   number = {28},
   pages = {11478-83},
   note = {Todorov, Emanuel
eng
Research Support, U.S. Gov't, Non-P.H.S.
Proc Natl Acad Sci U S A. 2009 Jul 14;106(28):11478-83. doi: 10.1073/pnas.0710743106. Epub 2009 Jul 2.},
   abstract = {Optimal choice of actions is a fundamental problem relevant to fields as diverse as neuroscience, psychology, economics, computer science, and control engineering. Despite this broad relevance the abstract setting is similar: we have an agent choosing actions over time, an uncertain dynamical system whose state is affected by those actions, and a performance criterion that the agent seeks to optimize. Solving problems of this kind remains hard, in part, because of overly generic formulations. Here, we propose a more structured formulation that greatly simplifies the construction of optimal control laws in both discrete and continuous domains. An exhaustive search over actions is avoided and the problem becomes linear. This yields algorithms that outperform Dynamic Programming and Reinforcement Learning, and thereby solve traditional problems more efficiently. Our framework also enables computations that were not possible before: composing optimal control laws by mixing primitives, applying deterministic methods to stochastic systems, quantifying the benefits of error tolerance, and inferring goals from behavioral data via convex optimization. Development of a general class of easily solvable problems tends to accelerate progress--as linear systems theory has done, for example. Our framework may have similar impact in fields where optimal choice of actions is relevant.},
   keywords = {*Algorithms
Information Science/*methods
*Models, Theoretical
*Problem Solving
Stochastic Processes
*Uncertainty},
   ISSN = {1091-6490 (Electronic)
0027-8424 (Linking)},
   DOI = {10.1073/pnas.0710743106},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/19574462
https://www.pnas.org/content/pnas/106/28/11478.full.pdf},
   year = {2009},
   type = {Journal Article}
}

@article{RN2646,
   author = {Turk, Matthew and Pentland, Alex},
   title = {Eigenfaces for recognition},
   journal = {Journal of Cognitive Neurosciece},
   volume = {3},
   number = {1},
   pages = {71-86},
   note = {Turk, M
Pentland, A
eng
J Cogn Neurosci. 1991 Winter;3(1):71-86. doi: 10.1162/jocn.1991.3.1.71.},
   abstract = {We have developed a near-real-time computer system that can locate and track a subject's head, and then recognize the person by comparing characteristics of the face to those of known individuals. The computational approach taken in this system is motivated by both physiology and information theory, as well as by the practical requirements of near-real-time performance and accuracy. Our approach treats the face recognition problem as an intrinsically two-dimensional (2-D) recognition problem rather than requiring recovery of three-dimensional geometry, taking advantage of the fact that faces are normally upright and thus may be described by a small set of 2-D characteristic views. The system functions by projecting face images onto a feature space that spans the significant variations among known face images. The significant features are known as "eigenfaces," because they are the eigenvectors (principal components) of the set of faces; they do not necessarily correspond to features such as eyes, ears, and noses. The projection operation characterizes an individual face by a weighted sum of the eigenface features, and so to recognize a particular face it is necessary only to compare these weights to those of known individuals. Some particular advantages of our approach are that it provides for the ability to learn and later recognize new faces in an unsupervised manner, and that it is easy to implement using a neural network architecture.},
   ISSN = {0898-929X (Print)
0898-929X (Linking)},
   DOI = {10.1162/jocn.1991.3.1.71},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/23964806},
   year = {1991},
   type = {Journal Article}
}

@article{RN3520,
   author = {Usher, M. and Cohen, J. D. and Servan-Schreiber, D. and Rajkowski, J. and Aston-Jones, G.},
   title = {The role of Locus Coeruleus in the regulation of cognitive performance},
   journal = {Science},
   volume = {283},
   pages = {549-554},
   DOI = {10.1126/science.283.5401.549},
   year = {1999},
   type = {Journal Article}
}

@article{RN1672,
   author = {von der Malsburg, C.},
   title = {Self-organization of orientation sensitive cells in the striate cortex},
   journal = {Kybernetik},
   volume = {14},
   number = {2},
   pages = {85-100},
   note = {von der Malsburg, C
eng
Germany
Kybernetik. 1973 Dec 31;14(2):85-100.},
   keywords = {Animals
Cats
Cybernetics
Feedback
Learning
*Models, Neurological
Neurons, Afferent/physiology
*Orientation
Synapses/physiology
Visual Cortex/*physiology},
   ISSN = {0023-5946 (Print)
0023-5946 (Linking)},
   DOI = {10.1007/BF00288907},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/4786750},
   year = {1973},
   type = {Journal Article}
}

@article{RN6608,
   author = {Wang, J. X. and Kurth-Nelson, Z. and Kumaran, D. and Tirumala, D. and Soyer, H. and Leibo, J. Z. and Hassabis, D. and Botvinick, M.},
   title = {Prefrontal cortex as a meta-reinforcement learning system},
   journal = {Nat Neurosci},
   volume = {21},
   number = {6},
   pages = {860-868},
   note = {Wang, Jane X
Kurth-Nelson, Zeb
Kumaran, Dharshan
Tirumala, Dhruva
Soyer, Hubert
Leibo, Joel Z
Hassabis, Demis
Botvinick, Matthew
eng
Nat Neurosci. 2018 Jun;21(6):860-868. doi: 10.1038/s41593-018-0147-8. Epub 2018 May 14.},
   abstract = {Over the past 20 years, neuroscience research on reward-based learning has converged on a canonical model, under which the neurotransmitter dopamine 'stamps in' associations between situations, actions and rewards by modulating the strength of synaptic connections between neurons. However, a growing number of recent findings have placed this standard model under strain. We now draw on recent advances in artificial intelligence to introduce a new theory of reward-based learning. Here, the dopamine system trains another part of the brain, the prefrontal cortex, to operate as its own free-standing learning system. This new perspective accommodates the findings that motivated the standard model, but also deals gracefully with a wider range of observations, providing a fresh foundation for future research.},
   keywords = {Algorithms
Animals
Artificial Intelligence
Computer Simulation
Dopamine/physiology
Humans
Learning/*physiology
Models, Neurological
Optogenetics
Prefrontal Cortex/*physiology
*Reinforcement (Psychology)
Reward},
   ISSN = {1546-1726 (Electronic)
1097-6256 (Linking)},
   DOI = {10.1038/s41593-018-0147-8},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/29760527},
   year = {2018},
   type = {Journal Article}
}

@article{RN6515,
   author = {Wang, X. J. and Yang, G. R.},
   title = {A disinhibitory circuit motif and flexible information routing in the brain},
   journal = {Curr Opin Neurobiol},
   volume = {49},
   pages = {75-83},
   note = {Wang, Xiao-Jing
Yang, Guangyu Robert
eng
Review
England
Curr Opin Neurobiol. 2018 Apr;49:75-83. doi: 10.1016/j.conb.2018.01.002. Epub 2018 Feb 4.},
   abstract = {In the mammalian neocortex, an area typically receives inputs from, and projects to, dozens of other areas. Mechanisms are needed to flexibly route information to the right place at the right time, which we term 'pathway gating'. For instance, a region in your brain that receives signals from both visual and auditory pathways may want to 'gate in' the visual pathway while 'gating out' the auditory pathway when you try to read a book surrounded by people in a noisy cafe. In this review, we marshall experimental and computational evidence in support of a circuit mechanism for flexible pathway gating realized by a disinhibitory motif. Moreover, recent work shows an increasing preponderance of this disinhibitory motif from sensory areas to association areas of the mammalian cortex. Pathway input gating is briefly compared with alternative or complementary gating mechanisms. Predictions and open questions for future research on this puzzle about the complex brain system will be discussed.},
   ISSN = {1873-6882 (Electronic)
0959-4388 (Linking)},
   DOI = {10.1016/j.conb.2018.01.002},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/29414069
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6599531/pdf/nihms-1036508.pdf},
   year = {2018},
   type = {Journal Article}
}

@book{RN12257,
   author = {Weibull, J.},
   title = {Evolutionary Game Theory.},
   publisher = {MIT Press},
   year = {1995},
   type = {Book}
}

@article{RN12246,
   author = {Wilson, H. R. and Cowan, J. D.},
   title = {Excitatory and inhibitory interactions in localized populations of model neurons},
   journal = {Biophys J},
   volume = {12},
   number = {1},
   pages = {1-24},
   note = {Wilson, H R
Cowan, J D
eng
1972/01/01
Biophys J. 1972 Jan;12(1):1-24. doi: 10.1016/S0006-3495(72)86068-5.},
   abstract = {Coupled nonlinear differential equations are derived for the dynamics of spatially localized populations containing both excitatory and inhibitory model neurons. Phase plane methods and numerical solutions are then used to investigate population responses to various types of stimuli. The results obtained show simple and multiple hysteresis phenomena and limit cycle activity. The latter is particularly interesting since the frequency of the limit cycle oscillation is found to be a monotonic function of stimulus intensity. Finally, it is proved that the existence of limit cycle dynamics in response to one class of stimuli implies the existence of multiple stable states and hysteresis in response to a different class of stimuli. The relation between these findings and a number of experiments is discussed.},
   keywords = {Evoked Potentials
Feedback
*Models, Neurological
Neural Inhibition
Neurons/*physiology
Periodicity
*Synaptic Transmission},
   ISSN = {0006-3495 (Print)
1542-0086 (Electronic)
0006-3495 (Linking)},
   DOI = {10.1016/S0006-3495(72)86068-5},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/4332108},
   year = {1972},
   type = {Journal Article}
}

@article{RN871,
   author = {Wolpert, D. M. and Doya, K. and Kawato, M.},
   title = {A unifying computational framework for motor control and social interaction},
   journal = {Philosophical Transactions of the Royal Society B-Biological Sciences},
   volume = {358},
   number = {1431},
   pages = {593-602},
   note = {Wolpert, Daniel M
Doya, Kenji
Kawato, Mitsuo
eng
England
Philos Trans R Soc Lond B Biol Sci. 2003 Mar 29;358(1431):593-602.},
   abstract = {Recent empirical studies have implicated the use of the motor system during action observation, imitation and social interaction. In this paper, we explore the computational parallels between the processes that occur in motor control and in action observation, imitation, social interaction and theory of mind. In particular, we examine the extent to which motor commands acting on the body can be equated with communicative signals acting on other people and suggest that computational solutions for motor control may have been extended to the domain of social interaction.},
   keywords = {Humans
Imitative Behavior
*Interpersonal Relations
*Models, Psychological
Social Behavior},
   ISSN = {0962-8436 (Print)
0962-8436 (Linking)},
   DOI = {10.1098/rstb.2002.1238},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/12689384},
   year = {2003},
   type = {Journal Article}
}

@article{RN3527,
   author = {Wolpert, D. M. and Miall, R. C. and Kawato, M.},
   title = {Internal models in the cerebellum},
   journal = {Trends in Cognitive Sciences},
   volume = {2},
   pages = {338-347},
   DOI = {10.1016/S1364-6613(98)01221-2},
   year = {1998},
   type = {Journal Article}
}

@article{RN5035,
   author = {Yamins, D. L. and Hong, H. and Cadieu, C. F. and Solomon, E. A. and Seibert, D. and DiCarlo, J. J.},
   title = {Performance-optimized hierarchical models predict neural responses in higher visual cortex},
   journal = {Proc Natl Acad Sci U S A},
   volume = {111},
   number = {23},
   pages = {8619-24},
   note = {Yamins, Daniel L K
Hong, Ha
Cadieu, Charles F
Solomon, Ethan A
Seibert, Darren
DiCarlo, James J
eng
F32 EY022845/EY/NEI NIH HHS/
R01-EY014970/EY/NEI NIH HHS/
Research Support, N.I.H., Extramural
Research Support, U.S. Gov't, Non-P.H.S.
2014/05/09 06:00
Proc Natl Acad Sci U S A. 2014 Jun 10;111(23):8619-24. doi: 10.1073/pnas.1403112111. Epub 2014 May 8.},
   abstract = {The ventral visual stream underlies key human visual object recognition abilities. However, neural encoding in the higher areas of the ventral stream remains poorly understood. Here, we describe a modeling approach that yields a quantitatively accurate model of inferior temporal (IT) cortex, the highest ventral cortical area. Using high-throughput computational techniques, we discovered that, within a class of biologically plausible hierarchical neural network models, there is a strong correlation between a model's categorization performance and its ability to predict individual IT neural unit response data. To pursue this idea, we then identified a high-performing neural network that matches human performance on a range of recognition tasks. Critically, even though we did not constrain this model to match neural data, its top output layer turns out to be highly predictive of IT spiking responses to complex naturalistic images at both the single site and population levels. Moreover, the model's intermediate layers are highly predictive of neural responses in the V4 cortex, a midlevel visual area that provides the dominant cortical input to IT. These results show that performance optimization--applied in a biologically appropriate model class--can be used to build quantitative predictive models of neural processing.},
   keywords = {Algorithms
Animals
Humans
Macaca mulatta/*physiology
*Models, Neurological
Nerve Net/physiology
*Neural Networks (Computer)
Photic Stimulation/methods
Psychomotor Performance/physiology
Recognition (Psychology)/physiology
Visual Cortex/*physiology
Visual Pathways/physiology
Visual Perception/physiology
array electrophysiology
computational neuroscience
computer vision},
   ISSN = {1091-6490 (Electronic)
0027-8424 (Linking)},
   DOI = {10.1073/pnas.1403112111},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/24812127},
   year = {2014},
   type = {Journal Article}
}

@article{RN7088,
   author = {Yang, G. R. and Joglekar, M. R. and Song, H. F. and Newsome, W. T. and Wang, X. J.},
   title = {Task representations in neural networks trained to perform many cognitive tasks},
   journal = {Nat Neurosci},
   volume = {22},
   number = {2},
   pages = {297-306},
   note = {Yang, Guangyu Robert
Joglekar, Madhura R
Song, H Francis
Newsome, William T
Wang, Xiao-Jing
eng
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Nat Neurosci. 2019 Feb;22(2):297-306. doi: 10.1038/s41593-018-0310-2. Epub 2019 Jan 14.},
   abstract = {The brain has the ability to flexibly perform many tasks, but the underlying mechanism cannot be elucidated in traditional experimental and modeling studies designed for one task at a time. Here, we trained single network models to perform 20 cognitive tasks that depend on working memory, decision making, categorization, and inhibitory control. We found that after training, recurrent units can develop into clusters that are functionally specialized for different cognitive processes, and we introduce a simple yet effective measure to quantify relationships between single-unit neural representations of tasks. Learning often gives rise to compositionality of task representations, a critical feature for cognitive flexibility, whereby one task can be performed by recombining instructions for other tasks. Finally, networks developed mixed task selectivity similar to recorded prefrontal neurons after learning multiple tasks sequentially with a continual-learning technique. This work provides a computational platform to investigate neural representations of many cognitive tasks.},
   keywords = {Brain/*physiology
Cognition/*physiology
Computer Simulation
Decision Making/physiology
Humans
Learning/*physiology
Memory, Short-Term/physiology
*Models, Neurological
*Neural Networks, Computer
Neurons/physiology},
   ISSN = {1546-1726 (Electronic)
1097-6256 (Linking)},
   DOI = {10.1038/s41593-018-0310-2},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/30643294},
   year = {2019},
   type = {Journal Article}
}

@article{RN2813,
   author = {Yoshida, W. and Dolan, R. J. and Friston, K. J.},
   title = {Game theory of mind},
   journal = {PLoS Comput Biol},
   volume = {4},
   number = {12},
   pages = {e1000254},
   note = {Yoshida, Wako
Dolan, Ray J
Friston, Karl J
eng
088130/Wellcome Trust/United Kingdom
Wellcome Trust/United Kingdom
PLoS Comput Biol. 2008 Dec;4(12):e1000254. doi: 10.1371/journal.pcbi.1000254. Epub 2008 Dec 26.},
   abstract = {This paper introduces a model of 'theory of mind', namely, how we represent the intentions and goals of others to optimise our mutual interactions. We draw on ideas from optimum control and game theory to provide a 'game theory of mind'. First, we consider the representations of goals in terms of value functions that are prescribed by utility or rewards. Critically, the joint value functions and ensuing behaviour are optimised recursively, under the assumption that I represent your value function, your representation of mine, your representation of my representation of yours, and so on ad infinitum. However, if we assume that the degree of recursion is bounded, then players need to estimate the opponent's degree of recursion (i.e., sophistication) to respond optimally. This induces a problem of inferring the opponent's sophistication, given behavioural exchanges. We show it is possible to deduce whether players make inferences about each other and quantify their sophistication on the basis of choices in sequential games. This rests on comparing generative models of choices with, and without, inference. Model comparison is demonstrated using simulated and real data from a 'stag-hunt'. Finally, we note that exactly the same sophisticated behaviour can be achieved by optimising the utility function itself (through prosocial utility), producing unsophisticated but apparently altruistic agents. This may be relevant ethologically in hierarchal game theory and coevolution.},
   keywords = {Animals
Cognition/*physiology
Computer Simulation
Decision Making/*physiology
Game Theory
Humans
*Models, Biological
Predatory Behavior/*physiology
*Reward},
   ISSN = {1553-7358 (Electronic)
1553-734X (Linking)},
   DOI = {10.1371/journal.pcbi.1000254},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/19112488},
   year = {2008},
   type = {Journal Article}
}

@article{RN2818,
   author = {Yu, A. J. and Dayan, P.},
   title = {Uncertainty, neuromodulation, and attention},
   journal = {Neuron},
   volume = {46},
   number = {4},
   pages = {681-92},
   note = {Yu, Angela J
Dayan, Peter
eng
Neuron. 2005 May 19;46(4):681-92.},
   abstract = {Uncertainty in various forms plagues our interactions with the environment. In a Bayesian statistical framework, optimal inference and prediction, based on unreliable observations in changing contexts, require the representation and manipulation of different forms of uncertainty. We propose that the neuromodulators acetylcholine and norepinephrine play a major role in the brain's implementation of these uncertainty computations. Acetylcholine signals expected uncertainty, coming from known unreliability of predictive cues within a context. Norepinephrine signals unexpected uncertainty, as when unsignaled context switches produce strongly unexpected observations. These uncertainty signals interact to enable optimal inference and learning in noisy and changeable environments. This formulation is consistent with a wealth of physiological, pharmacological, and behavioral data implicating acetylcholine and norepinephrine in specific aspects of a range of cognitive processes. Moreover, the model suggests a class of attentional cueing tasks that involve both neuromodulators and shows how their interactions may be part-antagonistic, part-synergistic.},
   keywords = {Acetylcholine/metabolism/pharmacology
Algorithms
Animals
Attention/drug effects/*physiology
Bayes Theorem
Computer Simulation
Dose-Response Relationship, Drug
Generalization (Psychology)/drug effects/physiology
Humans
Learning/drug effects/physiology
*Likelihood Functions
Maze Learning/drug effects/physiology
*Models, Neurological
Muscarinic Antagonists/pharmacology
Nicotine/pharmacology
Nicotinic Agonists/pharmacology
Norepinephrine/metabolism/pharmacology
Rats
Scopolamine Hydrobromide/pharmacology
*Signal Detection, Psychological
Time Factors
*Uncertainty},
   ISSN = {0896-6273 (Print)
0896-6273 (Linking)},
   DOI = {10.1016/j.neuron.2005.04.026},
   url = {http://www.ncbi.nlm.nih.gov/pubmed/15944135},
   year = {2005},
   type = {Journal Article}
}

@book{RN12256,
   author = {Yu, X and Gen, M.},
   title = {Introduction to Evolutionary Algorithms},
   publisher = {Springer},
   year = {2012},
   type = {Book}
}

@article{RN2839,
   author = {Zemel, R. S. and Dayan, P. and Pouget, A.},
   title = {Probabilistic interpretation of population codes},
   journal = {Neural Comput},
   volume = {10},
   number = {2},
   pages = {403-30},
   note = {Zemel, R S
Dayan, P
Pouget, A
eng
R29 MH 55541-01/MH/NIMH NIH HHS/
Research Support, Non-U.S. Gov't
Research Support, U.S. Gov't, Non-P.H.S.
Research Support, U.S. Gov't, P.H.S.
Review
1998/02/24
Neural Comput. 1998 Feb 15;10(2):403-30. doi: 10.1162/089976698300017818.},
   abstract = {We present a general encoding-decoding framework for interpreting the activity of a population of units. A standard population code interpretation method, the Poisson model, starts from a description as to how a single value of an underlying quantity can generate the activities of each unit in the population. In casting it in the encoding-decoding framework, we find that this model is too restrictive to describe fully the activities of units in population codes in higher processing areas, such as the medial temporal area. Under a more powerful model, the population activity can convey information not only about a single value of some quantity but also about its whole distribution, including its variance, and perhaps even the certainty the system has in the actual presence in the world of the entity generating this quantity. We propose a novel method for forming such probabilistic interpretations of population codes and compare it to the existing method.},
   keywords = {*Data Interpretation, Statistical
*Models, Neurological
*Models, Statistical
Neurons/*physiology
Poisson Distribution
*Probability},
   ISSN = {0899-7667 (Print)
0899-7667 (Linking)},
   DOI = {10.1162/089976698300017818},
   url = {https://www.ncbi.nlm.nih.gov/pubmed/9472488},
   year = {1998},
   type = {Journal Article}
}