CLAW Publications

Training Vision-Language Transformers from Captions Alone

ArXiv
Liangke Gui, Qiuyuan Huang, Alex Hauptmann,

Yonatan Bisk

@article{Gui2022,
    author = {Liangke Gui and Qiuyuan Huang and Alex Hauptmann and Yonatan Bisk and Jianfeng Gao},
    title = {{Training Vision-Language Transformers from Captions Alone}},
    journal = {ArXiv},
    year = {2022},
    url = {https://arxiv.org/abs/2205.09256},
}

Symmetric Machine Theory of Mind

The Thirty-ninth International Conference on Machine Learning
Melanie Sclar, Graham Neubig,

Yonatan Bisk

BibTeX

@inproceedings{Sclar2022,
    author = {Melanie Sclar and Graham Neubig and Yonatan Bisk},
    title = {{Symmetric Machine Theory of Mind}},
    booktitle = {The Thirty-ninth International Conference on Machine Learning},
    year = {2022},
}

A Framework for Learning to Request Rich and Contextually Useful Information from Humans

The Thirty-ninth International Conference on Machine Learning
Khanh Nguyen,

Yonatan Bisk

, Hal Daume III
Paper BibTeX

@inproceedings{Nguyen2022,
    author = {Khanh Nguyen and Yonatan Bisk and Hal Daume III},
    title = {{A Framework for Learning to Request Rich and Contextually Useful Information from Humans}},
    booktitle = {The Thirty-ninth International Conference on Machine Learning},
    year = {2022},
    url = {http://arxiv.org/abs/2110.08258},
}

Simulated Language Learning from Communicative Goals and Linguistic Input

Proceedings of the Annual Meeting of the Cognitive Science Society
Hao Zhu,

Yonatan Bisk

, Graham Neubig
BibTeX

@inproceedings{Zhu2022,
    author = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title = {{Simulated Language Learning from Communicative Goals and Linguistic Input}},
    booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society},
    year = {2022},
}

HEAR 2021: Holistic Evaluation of Audio Representations

Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track
Joseph Turian, Jordie Shier, Humair Raj Khan, Bhiksha Raj, Björn W. Schuller, Christian J. Steinmetz, Colin Malloy, George Tzanetakis, Gissel Velarde, Kirk McNally, Max Henry, Nicolas Pinto, Camille Noufi, Christian Clough, Dorien Herremans, Eduardo Fonseca, Jesse Engel, Justin Salamon, Philippe Esling, Pranay Manocha, Shinji Watanabe, Zeyu Jin,

Yonatan Bisk

Paper BibTeX

@inproceedings{Turian2022,
    author = {Joseph Turian and Jordie Shier and Humair Raj Khan and Bhiksha Raj and Björn W. Schuller and Christian J. Steinmetz and Colin Malloy and George Tzanetakis and Gissel Velarde and Kirk McNally and Max Henry and Nicolas Pinto and Camille Noufi and Christian Clough and Dorien Herremans and Eduardo Fonseca and Jesse Engel and Justin Salamon and Philippe Esling and Pranay Manocha and Shinji Watanabe and Zeyu Jin and Yonatan Bisk},
    title = {{HEAR 2021: Holistic Evaluation of Audio Representations}},
    booktitle = {Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track},
    year = {2022},
    url = {http://arxiv.org/abs/2203.03022},
}

KAT: A Knowledge Augmented Transformer for Vision-and-Language

Annual Conference of the North American Chapter of the Association for Computational Linguistics
Liangke Gui, Borui Wang, Qiuyuan Huang, Alex Hauptmann,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX

@inproceedings{Gui2022,
    author = {Liangke Gui and Borui Wang and Qiuyuan Huang and Alex Hauptmann and Yonatan Bisk and Jianfeng Gao},
    title = {{KAT: A Knowledge Augmented Transformer for Vision-and-Language}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year = {2022},
    url = {https://arxiv.org/abs/2112.08614},
}

FILM: Following Instructions in Language with Modular Methods

The Tenth International Conference on Learning Representations
So Yeon Min, Devendra Singh Chaplot, Pradeep Ravikumar,

Yonatan Bisk

, Ruslan Salakhutdinov
Paper BibTeX

@inproceedings{Min2022,
    author = {So Yeon Min and Devendra Singh Chaplot and Pradeep Ravikumar and Yonatan Bisk and Ruslan Salakhutdinov},
    title = {{FILM: Following Instructions in Language with Modular Methods}},
    booktitle = {The Tenth International Conference on Learning Representations},
    year = {2022},
    url = {https://soyeonm.github.io/FILM_webpage/},
}

WebQA: Multihop and Multimodal QA

Conference on Computer Vision and Pattern Recognition
Yingshan Chang, Mridu Narang, Hisami Suzuki, Guihong Cao, Jianfeng Gao,

Yonatan Bisk

Paper BibTeX

@inproceedings{Chang2022,
    author = {Yingshan Chang and Mridu Narang and Hisami Suzuki and Guihong Cao and Jianfeng Gao and Yonatan Bisk},
    title = {{WebQA: Multihop and Multimodal QA}},
    booktitle = {Conference on Computer Vision and Pattern Recognition},
    year = {2022},
    url = {https://arxiv.org/abs/2109.00590},
}

Dependency Induction Through the Lens of Visual Perception

The SIGNLL Conference on Computational Natural Language Learning
Ruisi Su, Shruti Rijhwani, Hao Zhu, Junxian He, Xinyu Wang,

Yonatan Bisk

, Graham Neubig
Paper BibTeX

@inproceedings{Su2021,
    author = {Ruisi Su and Shruti Rijhwani and Hao Zhu and Junxian He and Xinyu Wang and Yonatan Bisk and Graham Neubig},
    title = {{Dependency Induction Through the Lens of Visual Perception}},
    booktitle = {The SIGNLL Conference on Computational Natural Language Learning},
    year = {2021},
    url = {https://arxiv.org/abs/2109.09790},
}

Language Grounding with 3D Objects

Conference on Robot Learning
Jesse Thomason, Mohit Shridhar,

Yonatan Bisk

, Chris Paxton, Luke Zettlemoyer
Paper BibTeX

@inproceedings{Thomason2021,
    author = {Jesse Thomason and Mohit Shridhar and Yonatan Bisk and Chris Paxton and Luke Zettlemoyer},
    title = {{Language Grounding with 3D Objects}},
    booktitle = {Conference on Robot Learning},
    year = {2021},
    url = {https://arxiv.org/abs/2107.12514},
}

TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment

International Conference on Computer Vision
Jianwei Yang,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX

@inproceedings{Yang2021,
    author = {Jianwei Yang and Yonatan Bisk and Jianfeng Gao},
    title = {{TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment}},
    booktitle = {International Conference on Computer Vision},
    year = {2021},
    url = {https://arxiv.org/abs/2108.09980},
}

Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models

4th Workshop on Gender Bias in Natural Language Processing
Tejas Srinivasan,

Yonatan Bisk

Paper BibTeX

@inproceedings{Srinivasan2022,
    author = {Tejas Srinivasan and Yonatan Bisk},
    title = {{Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models}},
    booktitle = {4th Workshop on Gender Bias in Natural Language Processing},
    year = {2022},
    url = {https://arxiv.org/abs/2104.08666},
}

KB-VLP: Knowledge Based Vision and Language Pretraining

ICML21 Workshop on Self-Supervised Learning
Kezhen Chen, Qiuyuan Huang,

Yonatan Bisk

, Daniel McDuff, Jianfeng Gao
Paper BibTeX

@inproceedings{Chen2021,
    author = {Kezhen Chen and Qiuyuan Huang and Yonatan Bisk and Daniel McDuff and Jianfeng Gao},
    title = {{KB-VLP: Knowledge Based Vision and Language Pretraining}},
    booktitle = {ICML21 Workshop on Self-Supervised Learning},
    year = {2021},
    url = {https://www.microsoft.com/en-us/research/publication/kb-vlp-knowledge-based-vision-and-language-pretraining-2/},
}

Few-shot Language Coordination by Modeling Theory of Mind

The Thirty-eighth International Conference on Machine Learning
Hao Zhu, Graham Neubig,

Yonatan Bisk

Paper BibTeX

@inproceedings{Zhu2021,
    author = {Hao Zhu and Graham Neubig and Yonatan Bisk},
    title = {{Few-shot Language Coordination by Modeling Theory of Mind}},
    booktitle = {The Thirty-eighth International Conference on Machine Learning},
    year = {2021},
    url = {http://proceedings.mlr.press/v139/zhu21d.html},
}

Grounding `Grounding' in NLP

Findings of The 2021 Conference of the Association for Computational Linguistics
Khyathi Raghavi Chandu,

Yonatan Bisk

, Alan W Black
Paper BibTeX

@inproceedings{Chandu2021,
    author = {Khyathi Raghavi Chandu and Yonatan Bisk and Alan W Black},
    title = {{Grounding `Grounding' in NLP}},
    booktitle = {Findings of The 2021 Conference of the Association for Computational Linguistics},
    year = {2021},
    url = {https://arxiv.org/abs/2106.02192},
}

An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games

The 2021 Conference of the European Chapter of the Association for Computational Linguistics
Alessandro Suglia,

Yonatan Bisk

, Ioannis Konstas, Antonio Vergari, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX

@inproceedings{Suglia2021,
    author = {Alessandro Suglia and Yonatan Bisk and Ioannis Konstas and Antonio Vergari and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title = {{An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games}},
    booktitle = {The 2021 Conference of the European Chapter of the Association for Computational Linguistics},
    year = {2021},
    url = {https://arxiv.org/abs/2102.00424},
}

Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering

35th AAAI Conference on Artificial Intelligence
Kaixin Ma, Filip Ilievski, Jonathan Francis,

Yonatan Bisk

, Eric Nyberg, Alessandro Oltramari
Paper BibTeX Code

@inproceedings{Ma2021,
    author = {Kaixin Ma and Filip Ilievski and Jonathan Francis and Yonatan Bisk and Eric Nyberg and Alessandro Oltramari},
    title = {{Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering}},
    booktitle = {35th AAAI Conference on Artificial Intelligence},
    year = {2021},
    url = {https://arxiv.org/abs/2011.03863},
}

ALFWorld: Aligning Text and Embodied Environments for Interactive Learning

Proceedings of the International Conference on Learning Representations
Mohit Shridhar, Xingdi Yuan, Marc-Alexandre Côté,

Yonatan Bisk

, Adam Trischler, Matthew Hausknecht
Paper BibTeX

@inproceedings{Shridhar2021,
    author = {Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C{\^o}t{\'e} and Yonatan Bisk and Adam Trischler and Matthew Hausknecht},
    title = {{ALFWorld: Aligning Text and Embodied Environments for Interactive Learning}},
    booktitle = {Proceedings of the International Conference on Learning Representations},
    year = {2021},
    url = {https://alfworld.github.io/},
}

Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games

The 28th International Conference on Computational Linguistics
Alessandro Suglia, Antonio Vergari, Ioannis Konstas,

Yonatan Bisk

, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX

@inproceedings{Suglia2020,
    author = {Alessandro Suglia and Antonio Vergari and Ioannis Konstas and Yonatan Bisk and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title = {{Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games}},
    booktitle = {The 28th International Conference on Computational Linguistics},
    year = {2020},
    url = {https://arxiv.org/abs/2011.02917},
}

The Return of Lexical Dependencies: Neural Lexicalized PCFGs

Transactions of the Association for Computational Linguistics
Hao Zhu,

Yonatan Bisk

, Graham Neubig
Paper BibTeX Code

@article{Zhu2020,
    author = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title = {{The Return of Lexical Dependencies: Neural Lexicalized PCFGs}},
    journal = {Transactions of the Association for Computational Linguistics},
    year = {2020},
    url = {http://arxiv.org/abs/2007.15135},
}

Experience Grounds Language

Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)

Yonatan Bisk

, Ari Holtzman, Jesse Thomason, Jacob Andreas, Yoshua Bengio, Joyce Chai, Mirella Lapata, Angeliki Lazaridou, Jonathan May, Aleksandr Nisnevich, Nicolas Pinto, Joseph Turian
Paper BibTeX Video Slides

@inproceedings{Bisk2020,
    author = {Yonatan Bisk and Ari Holtzman and Jesse Thomason and Jacob Andreas and Yoshua Bengio and Joyce Chai and Mirella Lapata and Angeliki Lazaridou and Jonathan May and Aleksandr Nisnevich and Nicolas Pinto and Joseph Turian},
    title = {{Experience Grounds Language}},
    booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year = {2020},
    url = {https://arxiv.org/abs/2004.10151},
}

RMM: A Recursive Mental Model for Dialog Navigation

Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings
Homero Roman Roman,

Yonatan Bisk

, Jesse Thomason, Asli Celikyilmaz, Jianfeng Gao
Paper BibTeX Code

@inproceedings{Roman2020,
    author = {Homero Roman Roman and Yonatan Bisk and Jesse Thomason and Asli Celikyilmaz and Jianfeng Gao},
    title = {{RMM: A Recursive Mental Model for Dialog Navigation}},
    booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings},
    year = {2020},
    url = {https://arxiv.org/abs/2005.00728},
}

A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos

Workshop on NLP Beyond Text
Frank F. Xu, Lei Ji, Botian Shi, Junyi Du, Graham Neubig,

Yonatan Bisk

, Nan Duan
Paper BibTeX Code

@inproceedings{Xu2020,
    author = {Frank F. Xu and Lei Ji and Botian Shi and Junyi Du and Graham Neubig and Yonatan Bisk and Nan Duan},
    title = {{A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos}},
    booktitle = {Workshop on NLP Beyond Text},
    year = {2020},
    url = {https://arxiv.org/abs/2005.00706},
}

Multi-View Learning for Vision-and-Language Navigation

ArXiv
Qiaolin Xia, Xiujun Li, Chunyuan Li,

Yonatan Bisk

, Zhifang Sui, Jianfeng Gao, Yejin Choi, Noah A. Smith
Paper BibTeX

@article{Xia2020,
    author = {Qiaolin Xia and Xiujun Li and Chunyuan Li and Yonatan Bisk and Zhifang Sui and Jianfeng Gao and Yejin Choi and Noah A. Smith},
    title = {{Multi-View Learning for Vision-and-Language Navigation}},
    journal = {ArXiv},
    year = {2020},
    url = {https://arxiv.org/abs/2003.00857},
}

ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Mohit Shridhar, Jesse Thomason, Daniel Gordon,

Yonatan Bisk

, Winson Han, Roozbeh Mottaghi, Luke Zettlemoyer, Dieter Fox
Paper BibTeX

@inproceedings{Shridhar2020,
    author = {Mohit Shridhar and Jesse Thomason and Daniel Gordon and Yonatan Bisk and Winson Han and Roozbeh Mottaghi and Luke Zettlemoyer and Dieter Fox},
    title = {{ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year = {2020},
    url = {https://askforalfred.com/},
}

PIQA: Reasoning about Physical Commonsense in Natural Language

Thirty-Fourth AAAI Conference on Artificial Intelligence

Yonatan Bisk

, Rowan Zellers, Ronan Le Bras, Jianfeng Gao, Yejin Choi
Paper BibTeX Slides

@inproceedings{Bisk2020,
    author = {Yonatan Bisk and Rowan Zellers and Ronan Le Bras and Jianfeng Gao and Yejin Choi},
    title = {{PIQA: Reasoning about Physical Commonsense in Natural Language}},
    booktitle = {Thirty-Fourth AAAI Conference on Artificial Intelligence},
    year = {2020},
    url = {https://yonatanbisk.com/piqa},
}

Robust Navigation with Language Pretraining and Stochastic Sampling

Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing
Xiujun Li, Chunyuan Li, Qiaolin Xia,

Yonatan Bisk

, Asli Celikyilmaz, Jianfeng Gao, Noah A. Smith, Yejin Choi
Paper BibTeX

@inproceedings{Li2019,
    author = {Xiujun Li and Chunyuan Li and Qiaolin Xia and Yonatan Bisk and Asli Celikyilmaz and Jianfeng Gao and Noah A. Smith and Yejin Choi},
    title = {{Robust Navigation with Language Pretraining and Stochastic Sampling}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing},
    year = {2019},
    url = {https://arxiv.org/abs/1909.02244},
}

Defending Against Neural Fake News

Thirty-third Conference on Neural Information Processing Systems
Rowan Zellers, Ari Holtzman, Hannah Rashkin,

Yonatan Bisk

, Ali Farhadi, Franziska Roesner, Yejin Choi
Paper BibTeX Demo Models

@inproceedings{Zellers2019,
    author = {Rowan Zellers and Ari Holtzman and Hannah Rashkin and Yonatan Bisk and Ali Farhadi and Franziska Roesner and Yejin Choi},
    title = {{Defending Against Neural Fake News}},
    booktitle = {Thirty-third Conference on Neural Information Processing Systems},
    year = {2019},
    url = {https://arxiv.org/abs/1905.12616},
}

FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks

bioRxiv
Ranjani Murali, James Hemp, Victoria Orphan,

Yonatan Bisk

Paper BibTeX

@article{Murali2019,
    author = {Ranjani Murali and James Hemp and Victoria Orphan and Yonatan Bisk},
    title = {{FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks}},
    journal = {bioRxiv},
    year = {2019},
    url = {https://www.biorxiv.org/content/10.1101/592808v1},
}

Improving Robot Success Detection using Static Object Data

Proceedings of the 2019 IEEE/RSJ International Conference on Intelligent Robots and Systems
Rosario Scalise, Jesse Thomason,

Yonatan Bisk

, Siddhartha Srinivasa
Paper BibTeX Video

@inproceedings{Scalise2019,
    author = {Rosario Scalise and Jesse Thomason and Yonatan Bisk and Siddhartha Srinivasa},
    title = {{Improving Robot Success Detection using Static Object Data}},
    booktitle = {Proceedings of the 2019 IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year = {2019},
    url = {https://arxiv.org/abs/1904.01650},
}

Early Fusion for Goal Directed Robotic Vision

Proceedings of the 2019 IEEE/RSJ International Conference on Intelligent Robots and Systems
Aaron Walsman,

Yonatan Bisk

, Saadia Gabriel, Dipendra Misra, Yoav Artzi, Yejin Choi, Dieter Fox
Paper BibTeX

@inproceedings{Walsman2019,
    author = {Aaron Walsman and Yonatan Bisk and Saadia Gabriel and Dipendra Misra and Yoav Artzi and Yejin Choi and Dieter Fox},
    title = {{Early Fusion for Goal Directed Robotic Vision}},
    booktitle = {Proceedings of the 2019 IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year = {2019},
    url = {https://arxiv.org/abs/1811.08824},
}

HellaSwag: Can a Machine Really Finish Your Sentence?

Proceedings of the Association for Computational Linguistics
Rowan Zellers, Ari Holtzman,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX

@inproceedings{Zellers2019,
    author = {Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title = {{HellaSwag: Can a Machine Really Finish Your Sentence? }},
    booktitle = {Proceedings of the Association for Computational Linguistics},
    year = {2019},
    url = {https://arxiv.org/abs/1905.07830},
}

From Recognition to Cognition: Visual Commonsense Reasoning

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Rowan Zellers,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX

@inproceedings{Zellers2019,
    author = {Rowan Zellers and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title = {{From Recognition to Cognition: Visual Commonsense Reasoning}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year = {2019},
    url = {http://visualcommonsense.com/},
}

Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Liyiming Ke, Xiujun Li,

Yonatan Bisk

, Ari Holtzman, Zhe Gan, Jingjing Liu, Jianfeng Gao, Yejin Choi, Siddhartha Srinivasa
Paper BibTeX Video

@inproceedings{Ke2019,
    author = {Liyiming Ke and Xiujun Li and Yonatan Bisk and Ari Holtzman and Zhe Gan and Jingjing Liu and Jianfeng Gao and Yejin Choi and Siddhartha Srinivasa},
    title = {{Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year = {2019},
    url = {https://arxiv.org/abs/1903.02547},
}

Prospection: Interpretable Plans From Language By Predicting the Future

Proceedings of the International Conference on Robotics and Automation (ICRA)
Chris Paxton,

Yonatan Bisk

, Jesse Thomason, Arunkumar Byravan, Dieter Fox
Paper BibTeX

@inproceedings{Paxton2019,
    author = {Chris Paxton and Yonatan Bisk and Jesse Thomason and Arunkumar Byravan and Dieter Fox},
    title = {{Prospection: Interpretable Plans From Language By Predicting the Future}},
    booktitle = {Proceedings of the International Conference on Robotics and Automation (ICRA)},
    year = {2019},
    url = {https://arxiv.org/abs/1903.08309},
}

Shifting the Baseline: Single Modality Performance on Visual Navigation & QA

Proceedings of the Annual Conference of the North American Chapter of the Association for Computational Linguistics
Jesse Thomason, Daniel Gordon,

Yonatan Bisk

Paper BibTeX

@inproceedings{Thomason2019,
    author = {Jesse Thomason and Daniel Gordon and Yonatan Bisk},
    title = {{Shifting the Baseline: Single Modality Performance on Visual Navigation & QA}},
    booktitle = {Proceedings of the Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year = {2019},
    url = {https://arxiv.org/abs/1811.00613},
}

Benchmarking Hierarchical Script Knowledge

Proceedings of the Annual Conference of the North American Chapter of the Association for Computational Linguistics

Yonatan Bisk

, Jan Buys, Karl Pichotta, Yejin Choi
Paper BibTeX Code

@inproceedings{Bisk2019,
    author = {Yonatan Bisk and Jan Buys and Karl Pichotta and Yejin Choi},
    title = {{Benchmarking Hierarchical Script Knowledge}},
    booktitle = {Proceedings of the Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year = {2019},
    url = {https://www.aclweb.org/anthology/N19-1412},
}

Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors

The 41st Annual Meeting of the Cognitive Science Society
Michael Hahn, Frank Keller,

Yonatan Bisk

, Yonatan Belinkov
Paper BibTeX

@inproceedings{Hahn2019,
    author = {Michael Hahn and Frank Keller and Yonatan Bisk and Yonatan Belinkov},
    title = {{Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors}},
    booktitle = {The 41st Annual Meeting of the Cognitive Science Society},
    year = {2019},
    url = {https://psyarxiv.com/zcwgn/},
}

SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference

Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP)
Rowan Zellers,

Yonatan Bisk

, Roy Schwartz, Yejin Choi
Paper BibTeX Talk

@inproceedings{Zellers2018,
    author = {Rowan Zellers and Yonatan Bisk and Roy Schwartz and Yejin Choi},
    title = {{SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference}},
    booktitle = {Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year = {2018},
    url = {http://rowanzellers.com/swag/},
}

Inducing Grammars with and for Neural Machine Translation

Proceedings of the 2nd Workshop on Neural Machine Translation
Ke Tran,

Yonatan Bisk

Paper BibTeX

@inproceedings{Tran2018,
    author = {Ke Tran and Yonatan Bisk},
    title = {{Inducing Grammars with and for Neural Machine Translation}},
    booktitle = {Proceedings of the 2nd Workshop on Neural Machine Translation},
    year = {2018},
    url = {https://arxiv.org/abs/1805.10850},
}

Bridging HMMs and RNNs through Architectural Transformations

32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop
Jan Buys,

Yonatan Bisk

, Yejin Choi
Paper BibTeX

@inproceedings{Buys2018,
    author = {Jan Buys and Yonatan Bisk and Yejin Choi},
    title = {{Bridging HMMs and RNNs through Architectural Transformations}},
    booktitle = {32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop},
    year = {2018},
    url = {https://openreview.net/pdf?id=rJxEso0osm},
}

Synthetic and Natural Noise Both Break Neural Machine Translation

Proceedings of the 6th International Conference on Learning Representations
Yonatan Belinkov,

Yonatan Bisk

Paper BibTeX

@inproceedings{Belinkov2018,
    author = {Yonatan Belinkov and Yonatan Bisk},
    title = {{Synthetic and Natural Noise Both Break Neural Machine Translation }},
    booktitle = {Proceedings of the 6th International Conference on Learning Representations},
    year = {2018},
    url = {https://arxiv.org/abs/1711.02173},
}

CHALET: Cornell House Agent Learning Environment

ArXiv Preprint 1801.07357
Claudia Yan, Dipendra Misra, Andrew Bennett, Aaron Walsman,

Yonatan Bisk

, Yoav Artzi
Paper BibTeX Simulator

@article{Yan2018,
    author = {Claudia Yan and Dipendra Misra and Andrew Bennett and Aaron Walsman and Yonatan Bisk and Yoav Artzi},
    title = {{CHALET: Cornell House Agent Learning Environment }},
    journal = {ArXiv Preprint 1801.07357},
    year = {2018},
    url = {https://arxiv.org/abs/1801.07357},
}

Learning Interpretable Spatial Operations in a Rich 3D Blocks World

Proceedings of the Thirty-Second Conference on Artificial Intelligence (AAAI-18)

Yonatan Bisk

, Kevin Shih, Yejin Choi, Daniel Marcu
Paper BibTeX Data Poster

@inproceedings{Bisk2018,
    author = {Yonatan Bisk and Kevin Shih and Yejin Choi and Daniel Marcu},
    title = {{Learning Interpretable Spatial Operations in a Rich 3D Blocks World }},
    booktitle = {Proceedings of the Thirty-Second Conference on Artificial Intelligence (AAAI-18)},
    year = {2018},
    url = {https://arxiv.org/abs/1712.03463},
}

Natural Language Inference from Multiple Premises

Proceedings of the Eighth International Joint Conference on Natural Language Processing
Alice Lai,

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX

@inproceedings{Lai2017,
    author = {Alice Lai and Yonatan Bisk and Julia Hockenmaier},
    title = {{Natural Language Inference from Multiple Premises}},
    booktitle = {Proceedings of the Eighth International Joint Conference on Natural Language Processing},
    year = {2017},
    url = {https://arxiv.org/abs/1710.02925},
}

Natural Language Communication with Robots

Proceedings of the 15th Annual Conference of the North American Chapter of the Association for Computational Linguistics

Yonatan Bisk

, Deniz Yuret, Daniel Marcu
Paper BibTeX Data Slides

@inproceedings{Bisk2016,
    author = {Yonatan Bisk and Deniz Yuret and Daniel Marcu},
    title = {{Natural Language Communication with Robots }},
    booktitle = {Proceedings of the 15th Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year = {2016},
    url = {https://www.aclweb.org/anthology/N16-1089/},
}

Supertagging with LSTMs

Proceedings of the 15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers)
Ashish Vaswani,

Yonatan Bisk

, Kenji Sagae, Ryan Musa
Paper BibTeX

@inproceedings{Vaswani2016,
    author = {Ashish Vaswani and Yonatan Bisk and Kenji Sagae and Ryan Musa},
    title = {{Supertagging with LSTMs}},
    booktitle = {Proceedings of the 15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers)},
    year = {2016},
    url = {https://www.aclweb.org/anthology/N16-1027/},
}

Towards a Dataset for Human Computer Communication via Grounded Language Acquisition

Proceedings of the AAAI 2016 Workshop on Symbiotic Cognitive Systems

Yonatan Bisk

, Daniel Marcu, William Wong
Paper BibTeX

@inproceedings{Bisk2016,
    author = {Yonatan Bisk and Daniel Marcu and William Wong},
    title = {{Towards a Dataset for Human Computer Communication via Grounded Language Acquisition}},
    booktitle = {Proceedings of the AAAI 2016 Workshop on Symbiotic Cognitive Systems},
    year = {2016},
    url = {http://yonatanbisk.com/papers/2016-AAAI-Wksp.pdf},
}

Unsupervised Neural Hidden Markov Models

Proceedings of the Workshop on Structured Prediction for NLP
Ke Tran,

Yonatan Bisk

, Ashish Vaswani, Daniel Marcu, Kevin Knight
Paper BibTeX Slides

@inproceedings{Tran2016,
    author = {Ke Tran and Yonatan Bisk and Ashish Vaswani and Daniel Marcu and Kevin Knight},
    title = {{Unsupervised Neural Hidden Markov Models }},
    booktitle = {Proceedings of the Workshop on Structured Prediction for NLP},
    year = {2016},
    url = {https://www.aclweb.org/anthology/W16-5907/},
}

Evaluating Induced CCG Parsers on Grounded Semantic Parsing

Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing

Yonatan Bisk

, Siva Reddy, John Blitzer, Julia Hockenmaier, Mark Steedman
Paper BibTeX

@inproceedings{Bisk2016,
    author = {Yonatan Bisk and Siva Reddy and John Blitzer and Julia Hockenmaier and Mark Steedman},
    title = {{Evaluating Induced CCG Parsers on Grounded Semantic Parsing }},
    booktitle = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    year = {2016},
    url = {http://sivareddy.in/papers/bisk2016evaluating.pdf},
}

Unsupervised Grammar Induction with Combinatory Categorial Grammars

University of Illinois at Urbana-Champaign

Yonatan Bisk

Paper BibTeX

@phdthesis{Bisk2015,
    author = {Yonatan Bisk},
    title = {{Unsupervised Grammar Induction with Combinatory Categorial Grammars}},
    school = {University of Illinois at Urbana-Champaign},
    year = {2015},
    url = {http://yonatanbisk.com/papers/Thesis.pdf},
}

Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction

Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX

@inproceedings{Bisk2015,
    author = {Yonatan Bisk and Julia Hockenmaier},
    title = {{Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction}},
    booktitle = {Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    year = {2015},
    url = {https://www.aclweb.org/anthology/P15-1135/},
}

Labeled Grammar Induction with Minimal Supervision

Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)

Yonatan Bisk

, Christos Christodoulopoulos, Julia Hockenmaier
Paper BibTeX Poster

@inproceedings{Bisk2015,
    author = {Yonatan Bisk and Christos Christodoulopoulos and Julia Hockenmaier},
    title = {{Labeled Grammar Induction with Minimal Supervision }},
    booktitle = {Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)},
    year = {2015},
    url = {http://yonatanbisk.com/papers/2015-ACLShort-Fixed.pdf},
}

An HDP Model for Inducing Combinatory Categorial Grammars

Transactions of the Association for Computational Linguistics

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX Slides

@article{Bisk2013,
    author = {Yonatan Bisk and Julia Hockenmaier},
    title = {{An HDP Model for Inducing Combinatory Categorial Grammars}},
    journal = {Transactions of the Association for Computational Linguistics},
    year = {2013},
    url = {https://www.aclweb.org/anthology/Q13-1007/},
}

Simple Robust Grammar Induction with Combinatory Categorial Grammar

Proceedings of the Twenty-Sixth Conference on Artificial Intelligence (AAAI-12)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX

@inproceedings{Bisk2012,
    author = {Yonatan Bisk and Julia Hockenmaier},
    title = {{Simple Robust Grammar Induction with Combinatory Categorial Grammar }},
    booktitle = {Proceedings of the Twenty-Sixth Conference on Artificial Intelligence (AAAI-12)},
    year = {2012},
    url = {http://yonatanbisk.com/papers/2012-AAAI.pdf},
}

Document-Topic Hierarchies from Document Graphs

Proceedings of the 21st ACM international conference on Information and knowledge management (CIKM 2012)
Tim Weninger,

Yonatan Bisk

, Jiawei Han
Paper BibTeX

@inproceedings{Weninger2012,
    author = {Tim Weninger and Yonatan Bisk and Jiawei Han},
    title = {{Document-Topic Hierarchies from Document Graphs }},
    booktitle = {Proceedings of the 21st ACM international conference on Information and knowledge management (CIKM 2012)},
    year = {2012},
    url = {http://yonatanbisk.com/papers/2012-CIKM.pdf},
}

Induction of Linguistic Structure with Combinatory Categorial Grammars

NAACL HLT Workshop on Induction of Linguistic Structure

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX

@inproceedings{Bisk2012,
    author = {Yonatan Bisk and Julia Hockenmaier},
    title = {{Induction of Linguistic Structure with Combinatory Categorial Grammars }},
    booktitle = {NAACL HLT Workshop on Induction of Linguistic Structure},
    year = {2012},
    url = {http://yonatanbisk.com/papers/2012-NAACL-WILS.pdf},
}

Normal-form parsing for CCGs with generalized composition and type-raising

Proceedings of the 23rd International Conference on Computational Linguistics (Coling 2010)
Julia Hockenmaier,

Yonatan Bisk

Paper BibTeX

@inproceedings{Hockenmaier2010,
    author = {Julia Hockenmaier and Yonatan Bisk},
    title = {{Normal-form parsing for CCGs with generalized composition and type-raising}},
    booktitle = {Proceedings of the 23rd International Conference on Computational Linguistics (Coling 2010)},
    year = {2010},
    url = {https://www.aclweb.org/anthology/C10-1053/},
}

EGL: You can't learn language ...

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social

EGL: You can't learn language ...

... from the radio Text → Perception

... by watching TV Perception → Action

... by yourself Action → Social

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social